HMVMXR0.cpp@ 79232

Last change on this file since 79232 was 79232, checked in by vboxsync, 5 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Enable VMCS shadowing in the secondary processor-based VM-execution controls. (Disabled for now).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 717.1 KB

Line
1	/* $Id: HMVMXR0.cpp 79232 2019-06-19 06:44:39Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#ifdef VBOX_WITH_REM
39	# include <VBox/vmm/rem.h>
40	#endif
41	#include "HMInternal.h"
42	#include <VBox/vmm/vm.h>
43	#include <VBox/vmm/hmvmxinline.h>
44	#include "HMVMXR0.h"
45	#include "dtrace/VBoxVMM.h"
46
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
52	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
53	# define HMVMX_ALWAYS_TRAP_PF
54	# define HMVMX_ALWAYS_FLUSH_TLB
55	# define HMVMX_ALWAYS_SWAP_EFER
56	#endif
57
58
59	/*********************************************************************************************************************************
60	* Defined Constants And Macros *
61	*********************************************************************************************************************************/
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name HMVMX_READ_XXX
72	* Flags to skip redundant reads of some common VMCS fields that are not part of
73	* the guest-CPU or VCPU state but are needed while handling VM-exits.
74	*/
75	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
76	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
77	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
78	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
79	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
80	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
81	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
82	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
83	/** @} */
84
85	/**
86	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
87	* guest using hardware-assisted VMX.
88	*
89	* This excludes state like GPRs (other than RSP) which are always are
90	* swapped and restored across the world-switch and also registers like EFER,
91	* MSR which cannot be modified by the guest without causing a VM-exit.
92	*/
93	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
94	\| CPUMCTX_EXTRN_RFLAGS \
95	\| CPUMCTX_EXTRN_RSP \
96	\| CPUMCTX_EXTRN_SREG_MASK \
97	\| CPUMCTX_EXTRN_TABLE_MASK \
98	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
99	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
100	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
101	\| CPUMCTX_EXTRN_TSC_AUX \
102	\| CPUMCTX_EXTRN_OTHER_MSRS \
103	\| CPUMCTX_EXTRN_CR0 \
104	\| CPUMCTX_EXTRN_CR3 \
105	\| CPUMCTX_EXTRN_CR4 \
106	\| CPUMCTX_EXTRN_DR7 \
107	\| CPUMCTX_EXTRN_HM_VMX_MASK)
108
109	/**
110	* Exception bitmap mask for real-mode guests (real-on-v86).
111	*
112	* We need to intercept all exceptions manually except:
113	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
114	* due to bugs in Intel CPUs.
115	* - \#PF need not be intercepted even in real-mode if we have nested paging
116	* support.
117	*/
118	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
119	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
120	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
121	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
122	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
123	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
124	\| RT_BIT(X86_XCPT_XF))
125
126	/** Maximum VM-instruction error number. */
127	#define HMVMX_INSTR_ERROR_MAX 28
128
129	/** Profiling macro. */
130	#ifdef HM_PROFILE_EXIT_DISPATCH
131	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
132	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
133	#else
134	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
135	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
136	#endif
137
138	/** Assert that preemption is disabled or covered by thread-context hooks. */
139	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
140	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
141
142	/** Assert that we haven't migrated CPUs when thread-context hooks are not
143	* used. */
144	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
145	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
146	("Illegal migration! Entered on CPU %u Current %u\n", \
147	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
148
149	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
150	* context. */
151	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
152	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
153	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
154
155	/** Helper macro for VM-exit handlers called unexpectedly. */
156	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
157	do { \
158	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
159	return VERR_VMX_UNEXPECTED_EXIT; \
160	} while (0)
161
162	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
163	/** Macro that does the necessary privilege checks and intercepted VM-exits for
164	* guests that attempted to execute a VMX instruction. */
165	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
166	do \
167	{ \
168	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
169	if (rcStrictTmp == VINF_SUCCESS) \
170	{ /* likely */ } \
171	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
172	{ \
173	Assert((a_pVCpu)->hm.s.Event.fPending); \
174	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
175	return VINF_SUCCESS; \
176	} \
177	else \
178	{ \
179	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
180	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
181	} \
182	} while (0)
183
184	/** Macro that decodes a memory operand for an instruction VM-exit. */
185	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
186	do \
187	{ \
188	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
189	(a_pGCPtrEffAddr)); \
190	if (rcStrictTmp == VINF_SUCCESS) \
191	{ /* likely */ } \
192	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
193	{ \
194	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
195	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
196	NOREF(uXcptTmp); \
197	return VINF_SUCCESS; \
198	} \
199	else \
200	{ \
201	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
202	return rcStrictTmp; \
203	} \
204	} while (0)
205
206	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
207
208
209	/*********************************************************************************************************************************
210	* Structures and Typedefs *
211	*********************************************************************************************************************************/
212	/**
213	* VMX transient state.
214	*
215	* A state structure for holding miscellaneous information across
216	* VMX non-root operation and restored after the transition.
217	*/
218	typedef struct VMXTRANSIENT
219	{
220	/** The host's rflags/eflags. */
221	RTCCUINTREG fEFlags;
222	#if HC_ARCH_BITS == 32
223	uint32_t u32Alignment0;
224	#endif
225	/** The guest's TPR value used for TPR shadowing. */
226	uint8_t u8GuestTpr;
227	/** Alignment. */
228	uint8_t abAlignment0[7];
229
230	/** The basic VM-exit reason. */
231	uint16_t uExitReason;
232	/** Alignment. */
233	uint16_t u16Alignment0;
234	/** The VM-exit interruption error code. */
235	uint32_t uExitIntErrorCode;
236	/** The VM-exit exit code qualification. */
237	uint64_t uExitQual;
238	/** The Guest-linear address. */
239	uint64_t uGuestLinearAddr;
240
241	/** The VM-exit interruption-information field. */
242	uint32_t uExitIntInfo;
243	/** The VM-exit instruction-length field. */
244	uint32_t cbInstr;
245	/** The VM-exit instruction-information field. */
246	VMXEXITINSTRINFO ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Whether we are currently executing a nested-guest. */
250	bool fIsNestedGuest;
251	/** Alignment. */
252	uint8_t abAlignment1[2];
253
254	/** The VM-entry interruption-information field. */
255	uint32_t uEntryIntInfo;
256	/** The VM-entry exception error code field. */
257	uint32_t uEntryXcptErrorCode;
258	/** The VM-entry instruction length field. */
259	uint32_t cbEntryInstr;
260
261	/** IDT-vectoring information field. */
262	uint32_t uIdtVectoringInfo;
263	/** IDT-vectoring error code. */
264	uint32_t uIdtVectoringErrorCode;
265
266	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
267	uint32_t fVmcsFieldsRead;
268
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
274	bool fUpdatedTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringDoublePF;
278	/** Whether the VM-exit was caused by a page-fault during delivery of an
279	* external interrupt or NMI. */
280	bool fVectoringPF;
281	bool afAlignment0[3];
282
283	/** The VMCS info. object. */
284	PVMXVMCSINFO pVmcsInfo;
285	} VMXTRANSIENT;
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
290	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
291	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
292	/** Pointer to VMX transient state. */
293	typedef VMXTRANSIENT *PVMXTRANSIENT;
294	/** Pointer to a const VMX transient state. */
295	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
296
297
298	/**
299	* Memory operand read or write access.
300	*/
301	typedef enum VMXMEMACCESS
302	{
303	VMXMEMACCESS_READ = 0,
304	VMXMEMACCESS_WRITE = 1
305	} VMXMEMACCESS;
306
307	/**
308	* VMX VM-exit handler.
309	*
310	* @returns Strict VBox status code (i.e. informational status codes too).
311	* @param pVCpu The cross context virtual CPU structure.
312	* @param pVmxTransient The VMX-transient structure.
313	*/
314	#ifndef HMVMX_USE_FUNCTION_TABLE
315	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
316	#else
317	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
318	/** Pointer to VM-exit handler. */
319	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
320	#endif
321
322	/**
323	* VMX VM-exit handler, non-strict status code.
324	*
325	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
326	*
327	* @returns VBox status code, no informational status code returned.
328	* @param pVCpu The cross context virtual CPU structure.
329	* @param pVmxTransient The VMX-transient structure.
330	*
331	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
332	* use of that status code will be replaced with VINF_EM_SOMETHING
333	* later when switching over to IEM.
334	*/
335	#ifndef HMVMX_USE_FUNCTION_TABLE
336	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
337	#else
338	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
339	#endif
340
341
342	/*********************************************************************************************************************************
343	* Internal Functions *
344	*********************************************************************************************************************************/
345	#ifndef HMVMX_USE_FUNCTION_TABLE
346	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
347	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
348	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
349	#else
350	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
351	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
352	#endif
353	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
354	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
355	#endif
356
357	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
358	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
359	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
360	#endif
361
362	/** @name VM-exit handlers.
363	* @{
364	*/
365	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
366	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
367	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
389	#endif
390	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
391	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
392	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
393	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
394	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
395	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitMwait;
397	static FNVMXEXITHANDLER hmR0VmxExitMtf;
398	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
399	static FNVMXEXITHANDLER hmR0VmxExitPause;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
401	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
402	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
403	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
404	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
405	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
406	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
407	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
408	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
412	/** @} */
413
414	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
415	/** @name Nested-guest VM-exit handlers.
416	* @{
417	*/
418	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
419	//static FNVMXEXITHANDLER hmR0VmxExitExtIntNested;
420	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
422	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
423	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
424	//static FNVMXEXITHANDLER hmR0VmxExitCpuid;
425	//static FNVMXEXITHANDLER hmR0VmxExitGetsec;
426	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
427	//static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
428	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
429	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
430	//static FNVMXEXITHANDLER hmR0VmxExitVmcall;
431	//static FNVMXEXITHANDLER hmR0VmxExitVmclear;
432	//static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
433	//static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
434	//static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
435	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
436	//static FNVMXEXITHANDLER hmR0VmxExitVmresume;
437	//static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
438	//static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
439	//static FNVMXEXITHANDLER hmR0VmxExitVmxon;
440	//static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
441	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
442	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
443	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
444	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
445	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
446	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
447	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
448	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
449	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
450	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
451	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
452	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
453	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
454	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
455	//static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
456	//static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
457	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
458	//static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
459	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
460	//static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
461	//static FNVMXEXITHANDLER hmR0VmxExitErrUnexpected;
462	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
463	//static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
464	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
465	//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
466	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
467	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
468	/** @} */
469	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
470
471	/** @name Helpers for hardware exceptions VM-exit handlers.
472	* @{
473	*/
474	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
475	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
476	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
477	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
478	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
479	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
480	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
481	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
482	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr);
483	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
484	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
485	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu);
486	/** @} */
487
488
489	/*********************************************************************************************************************************
490	* Global Variables *
491	*********************************************************************************************************************************/
492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
493	/**
494	* Array of all VMCS fields.
495	* Any fields added to the VT-x spec. should be added here.
496	*
497	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
498	* of nested-guests.
499	*/
500	static const uint32_t g_aVmcsFields[] =
501	{
502	/* 16-bit control fields. */
503	VMX_VMCS16_VPID,
504	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
505	VMX_VMCS16_EPTP_INDEX,
506
507	/* 16-bit guest-state fields. */
508	VMX_VMCS16_GUEST_ES_SEL,
509	VMX_VMCS16_GUEST_CS_SEL,
510	VMX_VMCS16_GUEST_SS_SEL,
511	VMX_VMCS16_GUEST_DS_SEL,
512	VMX_VMCS16_GUEST_FS_SEL,
513	VMX_VMCS16_GUEST_GS_SEL,
514	VMX_VMCS16_GUEST_LDTR_SEL,
515	VMX_VMCS16_GUEST_TR_SEL,
516	VMX_VMCS16_GUEST_INTR_STATUS,
517	VMX_VMCS16_GUEST_PML_INDEX,
518
519	/* 16-bits host-state fields. */
520	VMX_VMCS16_HOST_ES_SEL,
521	VMX_VMCS16_HOST_CS_SEL,
522	VMX_VMCS16_HOST_SS_SEL,
523	VMX_VMCS16_HOST_DS_SEL,
524	VMX_VMCS16_HOST_FS_SEL,
525	VMX_VMCS16_HOST_GS_SEL,
526	VMX_VMCS16_HOST_TR_SEL,
527
528	/* 64-bit control fields. */
529	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
530	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
531	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
532	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
533	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
534	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
535	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
536	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
537	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
538	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
539	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
540	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
541	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
542	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
543	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
544	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
545	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
546	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
547	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
548	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
549	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
550	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
551	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
552	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
553	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
554	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
555	VMX_VMCS64_CTRL_EPTP_FULL,
556	VMX_VMCS64_CTRL_EPTP_HIGH,
557	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
558	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
559	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
560	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
561	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
562	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
563	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
564	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
565	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
566	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
567	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
568	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
569	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
570	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
571	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
572	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
573	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
574	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
575	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
576	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
577	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
578	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
579
580	/* 64-bit read-only data fields. */
581	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
582	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
583
584	/* 64-bit guest-state fields. */
585	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
586	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
587	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
588	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
589	VMX_VMCS64_GUEST_PAT_FULL,
590	VMX_VMCS64_GUEST_PAT_HIGH,
591	VMX_VMCS64_GUEST_EFER_FULL,
592	VMX_VMCS64_GUEST_EFER_HIGH,
593	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
594	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
595	VMX_VMCS64_GUEST_PDPTE0_FULL,
596	VMX_VMCS64_GUEST_PDPTE0_HIGH,
597	VMX_VMCS64_GUEST_PDPTE1_FULL,
598	VMX_VMCS64_GUEST_PDPTE1_HIGH,
599	VMX_VMCS64_GUEST_PDPTE2_FULL,
600	VMX_VMCS64_GUEST_PDPTE2_HIGH,
601	VMX_VMCS64_GUEST_PDPTE3_FULL,
602	VMX_VMCS64_GUEST_PDPTE3_HIGH,
603	VMX_VMCS64_GUEST_BNDCFGS_FULL,
604	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
605
606	/* 64-bit host-state fields. */
607	VMX_VMCS64_HOST_PAT_FULL,
608	VMX_VMCS64_HOST_PAT_HIGH,
609	VMX_VMCS64_HOST_EFER_FULL,
610	VMX_VMCS64_HOST_EFER_HIGH,
611	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
612	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
613
614	/* 32-bit control fields. */
615	VMX_VMCS32_CTRL_PIN_EXEC,
616	VMX_VMCS32_CTRL_PROC_EXEC,
617	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
618	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
619	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
620	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
621	VMX_VMCS32_CTRL_EXIT,
622	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
623	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
624	VMX_VMCS32_CTRL_ENTRY,
625	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
626	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
627	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
628	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
629	VMX_VMCS32_CTRL_TPR_THRESHOLD,
630	VMX_VMCS32_CTRL_PROC_EXEC2,
631	VMX_VMCS32_CTRL_PLE_GAP,
632	VMX_VMCS32_CTRL_PLE_WINDOW,
633
634	/* 32-bits read-only fields. */
635	VMX_VMCS32_RO_VM_INSTR_ERROR,
636	VMX_VMCS32_RO_EXIT_REASON,
637	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
638	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
639	VMX_VMCS32_RO_IDT_VECTORING_INFO,
640	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
641	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
642	VMX_VMCS32_RO_EXIT_INSTR_INFO,
643
644	/* 32-bit guest-state fields. */
645	VMX_VMCS32_GUEST_ES_LIMIT,
646	VMX_VMCS32_GUEST_CS_LIMIT,
647	VMX_VMCS32_GUEST_SS_LIMIT,
648	VMX_VMCS32_GUEST_DS_LIMIT,
649	VMX_VMCS32_GUEST_FS_LIMIT,
650	VMX_VMCS32_GUEST_GS_LIMIT,
651	VMX_VMCS32_GUEST_LDTR_LIMIT,
652	VMX_VMCS32_GUEST_TR_LIMIT,
653	VMX_VMCS32_GUEST_GDTR_LIMIT,
654	VMX_VMCS32_GUEST_IDTR_LIMIT,
655	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
656	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
657	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
658	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
659	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
660	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
661	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
662	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
663	VMX_VMCS32_GUEST_INT_STATE,
664	VMX_VMCS32_GUEST_ACTIVITY_STATE,
665	VMX_VMCS32_GUEST_SMBASE,
666	VMX_VMCS32_GUEST_SYSENTER_CS,
667	VMX_VMCS32_PREEMPT_TIMER_VALUE,
668
669	/* 32-bit host-state fields. */
670	VMX_VMCS32_HOST_SYSENTER_CS,
671
672	/* Natural-width control fields. */
673	VMX_VMCS_CTRL_CR0_MASK,
674	VMX_VMCS_CTRL_CR4_MASK,
675	VMX_VMCS_CTRL_CR0_READ_SHADOW,
676	VMX_VMCS_CTRL_CR4_READ_SHADOW,
677	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
678	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
679	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
680	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
681
682	/* Natural-width read-only data fields. */
683	VMX_VMCS_RO_EXIT_QUALIFICATION,
684	VMX_VMCS_RO_IO_RCX,
685	VMX_VMCS_RO_IO_RSI,
686	VMX_VMCS_RO_IO_RDI,
687	VMX_VMCS_RO_IO_RIP,
688	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
689
690	/* Natural-width guest-state field */
691	VMX_VMCS_GUEST_CR0,
692	VMX_VMCS_GUEST_CR3,
693	VMX_VMCS_GUEST_CR4,
694	VMX_VMCS_GUEST_ES_BASE,
695	VMX_VMCS_GUEST_CS_BASE,
696	VMX_VMCS_GUEST_SS_BASE,
697	VMX_VMCS_GUEST_DS_BASE,
698	VMX_VMCS_GUEST_FS_BASE,
699	VMX_VMCS_GUEST_GS_BASE,
700	VMX_VMCS_GUEST_LDTR_BASE,
701	VMX_VMCS_GUEST_TR_BASE,
702	VMX_VMCS_GUEST_GDTR_BASE,
703	VMX_VMCS_GUEST_IDTR_BASE,
704	VMX_VMCS_GUEST_DR7,
705	VMX_VMCS_GUEST_RSP,
706	VMX_VMCS_GUEST_RIP,
707	VMX_VMCS_GUEST_RFLAGS,
708	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
709	VMX_VMCS_GUEST_SYSENTER_ESP,
710	VMX_VMCS_GUEST_SYSENTER_EIP,
711
712	/* Natural-width host-state fields */
713	VMX_VMCS_HOST_CR0,
714	VMX_VMCS_HOST_CR3,
715	VMX_VMCS_HOST_CR4,
716	VMX_VMCS_HOST_FS_BASE,
717	VMX_VMCS_HOST_GS_BASE,
718	VMX_VMCS_HOST_TR_BASE,
719	VMX_VMCS_HOST_GDTR_BASE,
720	VMX_VMCS_HOST_IDTR_BASE,
721	VMX_VMCS_HOST_SYSENTER_ESP,
722	VMX_VMCS_HOST_SYSENTER_EIP,
723	VMX_VMCS_HOST_RSP,
724	VMX_VMCS_HOST_RIP
725	};
726	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
727
728	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
729	static const uint32_t g_aVmcsCacheSegBase[] =
730	{
731	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
732	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
733	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
734	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
735	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
736	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
737	};
738	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
739	#endif
740	static const uint32_t g_aVmcsSegBase[] =
741	{
742	VMX_VMCS_GUEST_ES_BASE,
743	VMX_VMCS_GUEST_CS_BASE,
744	VMX_VMCS_GUEST_SS_BASE,
745	VMX_VMCS_GUEST_DS_BASE,
746	VMX_VMCS_GUEST_FS_BASE,
747	VMX_VMCS_GUEST_GS_BASE
748	};
749	static const uint32_t g_aVmcsSegSel[] =
750	{
751	VMX_VMCS16_GUEST_ES_SEL,
752	VMX_VMCS16_GUEST_CS_SEL,
753	VMX_VMCS16_GUEST_SS_SEL,
754	VMX_VMCS16_GUEST_DS_SEL,
755	VMX_VMCS16_GUEST_FS_SEL,
756	VMX_VMCS16_GUEST_GS_SEL
757	};
758	static const uint32_t g_aVmcsSegLimit[] =
759	{
760	VMX_VMCS32_GUEST_ES_LIMIT,
761	VMX_VMCS32_GUEST_CS_LIMIT,
762	VMX_VMCS32_GUEST_SS_LIMIT,
763	VMX_VMCS32_GUEST_DS_LIMIT,
764	VMX_VMCS32_GUEST_FS_LIMIT,
765	VMX_VMCS32_GUEST_GS_LIMIT
766	};
767	static const uint32_t g_aVmcsSegAttr[] =
768	{
769	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
770	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
771	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
772	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
773	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
774	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
775	};
776	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
777	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
778	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
779	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
780
781	#ifdef HMVMX_USE_FUNCTION_TABLE
782	/**
783	* VMX_EXIT dispatch table.
784	*/
785	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
786	{
787	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
788	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
789	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
790	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
791	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
792	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
793	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
794	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
795	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
796	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
797	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
798	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
799	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
800	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
801	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
802	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
803	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
804	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
805	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
806	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
807	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
808	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
809	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
810	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
811	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
812	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
813	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
814	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
815	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
816	#else
817	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
818	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
819	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
820	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
821	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
822	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
823	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
824	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
825	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
826	#endif
827	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
828	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
829	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
830	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
831	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
832	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
833	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
834	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
835	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
836	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
837	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
838	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
839	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
840	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
841	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
842	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
843	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
844	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
845	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
846	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
847	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
848	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
849	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
850	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
851	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
852	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
853	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
854	#else
855	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
856	#endif
857	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
858	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
859	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
860	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
861	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
862	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
863	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
864	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
865	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
866	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
867	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
868	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
869	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
870	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
871	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
872	};
873	#endif /* HMVMX_USE_FUNCTION_TABLE */
874
875	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
876	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
877	{
878	/* 0 */ "(Not Used)",
879	/* 1 */ "VMCALL executed in VMX root operation.",
880	/* 2 */ "VMCLEAR with invalid physical address.",
881	/* 3 */ "VMCLEAR with VMXON pointer.",
882	/* 4 */ "VMLAUNCH with non-clear VMCS.",
883	/* 5 */ "VMRESUME with non-launched VMCS.",
884	/* 6 */ "VMRESUME after VMXOFF",
885	/* 7 */ "VM-entry with invalid control fields.",
886	/* 8 */ "VM-entry with invalid host state fields.",
887	/* 9 */ "VMPTRLD with invalid physical address.",
888	/* 10 */ "VMPTRLD with VMXON pointer.",
889	/* 11 */ "VMPTRLD with incorrect revision identifier.",
890	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
891	/* 13 */ "VMWRITE to read-only VMCS component.",
892	/* 14 */ "(Not Used)",
893	/* 15 */ "VMXON executed in VMX root operation.",
894	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
895	/* 17 */ "VM-entry with non-launched executing VMCS.",
896	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
897	/* 19 */ "VMCALL with non-clear VMCS.",
898	/* 20 */ "VMCALL with invalid VM-exit control fields.",
899	/* 21 */ "(Not Used)",
900	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
901	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
902	/* 24 */ "VMCALL with invalid SMM-monitor features.",
903	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
904	/* 26 */ "VM-entry with events blocked by MOV SS.",
905	/* 27 */ "(Not Used)",
906	/* 28 */ "Invalid operand to INVEPT/INVVPID."
907	};
908	#endif /* VBOX_STRICT */
909
910
911	/**
912	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
913	*
914	* Any bit set in this mask is owned by the host/hypervisor and would cause a
915	* VM-exit when modified by the guest.
916	*
917	* @returns The static CR0 guest/host mask.
918	* @param pVCpu The cross context virtual CPU structure.
919	*/
920	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
921	{
922	/*
923	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
924	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
925	*/
926	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
927	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
928	* and @bugref{6944}. */
929	PVM pVM = pVCpu->CTX_SUFF(pVM);
930	return ( X86_CR0_PE
931	\| X86_CR0_NE
932	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
933	\| X86_CR0_PG
934	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
935	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
936	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
937	}
938
939
940	/**
941	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
942	*
943	* Any bit set in this mask is owned by the host/hypervisor and would cause a
944	* VM-exit when modified by the guest.
945	*
946	* @returns The static CR4 guest/host mask.
947	* @param pVCpu The cross context virtual CPU structure.
948	*/
949	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
950	{
951	/*
952	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
953	* these bits are reserved on hardware that does not support them. Since the
954	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
955	* these bits and handle it depending on whether we expose them to the guest.
956	*/
957	PVM pVM = pVCpu->CTX_SUFF(pVM);
958	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
959	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
960	return ( X86_CR4_VMXE
961	\| X86_CR4_VME
962	\| X86_CR4_PAE
963	\| X86_CR4_PGE
964	\| X86_CR4_PSE
965	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
966	\| (fPcid ? X86_CR4_PCIDE : 0));
967	}
968
969
970	/**
971	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
972	* area.
973	*
974	* @returns @c true if it's different, @c false otherwise.
975	* @param pVmcsInfo The VMCS info. object.
976	*/
977	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
978	{
979	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
980	&& pVmcsInfo->pvGuestMsrStore);
981	}
982
983
984	/**
985	* Checks whether one of the given Pin-based VM-execution controls are set.
986	*
987	* @returns @c true if set, @c false otherwise.
988	* @param pVCpu The cross context virtual CPU structure.
989	* @param pVmxTransient The VMX-transient structure.
990	* @param uPinCtls The Pin-based VM-execution controls to check.
991	*
992	* @remarks This will not check merged controls when executing a nested-guest
993	* but the original control specified by the guest hypervisor.
994	*/
995	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
996	{
997	if (!pVmxTransient->fIsNestedGuest)
998	{
999	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1000	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
1001	}
1002	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
1003	}
1004
1005
1006	/**
1007	* Checks whether one of the given Processor-based VM-execution controls are set.
1008	*
1009	* @returns @c true if set, @c false otherwise.
1010	* @param pVCpu The cross context virtual CPU structure.
1011	* @param pVmxTransient The VMX-transient structure.
1012	* @param uProcCtls The Processor-based VM-execution controls to check.
1013	*
1014	* @remarks This will not check merged controls when executing a nested-guest
1015	* but the original control specified by the guest hypervisor.
1016	*/
1017	static bool hmR0VmxIsProcCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
1018	{
1019	if (!pVmxTransient->fIsNestedGuest)
1020	{
1021	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1022	return RT_BOOL(pVmcsInfo->u32ProcCtls & uProcCtls);
1023	}
1024	return CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
1025	}
1026
1027
1028	/**
1029	* Checks whether one of the given Secondary Processor-based VM-execution controls
1030	* are set.
1031	*
1032	* @returns @c true if set, @c false otherwise.
1033	* @param pVCpu The cross context virtual CPU structure.
1034	* @param pVmxTransient The VMX-transient structure.
1035	* @param uProcCtls2 The Secondary Processor-based VM-execution controls to
1036	* check.
1037	*
1038	* @remarks This will not check merged controls when executing a nested-guest
1039	* but the original control specified by the guest hypervisor.
1040	*/
1041	static bool hmR0VmxIsProcCtls2Set(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls2)
1042	{
1043	if (!pVmxTransient->fIsNestedGuest)
1044	{
1045	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1046	return RT_BOOL(pVmcsInfo->u32ProcCtls2 & uProcCtls2);
1047	}
1048	return CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls2);
1049	}
1050
1051
1052	#if 0
1053	/**
1054	* Checks whether one of the given VM-entry controls are set.
1055	*
1056	* @returns @c true if set, @c false otherwise.
1057	* @param pVCpu The cross context virtual CPU structure.
1058	* @param pVmxTransient The VMX-transient structure.
1059	* @param uEntryCtls The VM-entry controls to check.
1060	*
1061	* @remarks This will not check merged controls when executing a nested-guest
1062	* but the original control specified by the guest hypervisor.
1063	*/
1064	static bool hmR0VmxIsEntryCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uEntryCtls)
1065	{
1066	if (!pVmxTransient->fIsNestedGuest)
1067	{
1068	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1069	return RT_BOOL(pVmcsInfo->u32EntryCtls & uEntryCtls);
1070	}
1071	return CPUMIsGuestVmxEntryCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uEntryCtls);
1072	}
1073
1074
1075	/**
1076	* Checks whether one of the given VM-exit controls are set.
1077	*
1078	* @returns @c true if set, @c false otherwise.
1079	* @param pVCpu The cross context virtual CPU structure.
1080	* @param pVmxTransient The VMX-transient structure.
1081	* @param uExitCtls The VM-exit controls to check.
1082	*
1083	* @remarks This will not check merged controls when executing a nested-guest
1084	* but the original control specified by the guest hypervisor.
1085	*/
1086	static bool hmR0VmxIsExitCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uExitCtls)
1087	{
1088	if (!pVmxTransient->fIsNestedGuest)
1089	{
1090	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1091	return RT_BOOL(pVmcsInfo->u32ExitCtls & uExitCtls);
1092	}
1093	return CPUMIsGuestVmxExitCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uExitCtls);
1094	}
1095	#endif
1096
1097
1098	/**
1099	* Adds one or more exceptions to the exception bitmap and commits it to the current
1100	* VMCS.
1101	*
1102	* @returns VBox status code.
1103	* @param pVmxTransient The VMX-transient structure.
1104	* @param uXcptMask The exception(s) to add.
1105	*/
1106	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1107	{
1108	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1109	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1110	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1111	{
1112	uXcptBitmap \|= uXcptMask;
1113	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1114	AssertRCReturn(rc, rc);
1115	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1116	}
1117	return VINF_SUCCESS;
1118	}
1119
1120
1121	/**
1122	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1123	*
1124	* @returns VBox status code.
1125	* @param pVmxTransient The VMX-transient structure.
1126	* @param uXcpt The exception to add.
1127	*/
1128	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1129	{
1130	Assert(uXcpt <= X86_XCPT_LAST);
1131	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1132	}
1133
1134
1135	/**
1136	* Remove one or more exceptions from the exception bitmap and commits it to the
1137	* current VMCS.
1138	*
1139	* This takes care of not removing the exception intercept if a nested-guest
1140	* requires the exception to be intercepted.
1141	*
1142	* @returns VBox status code.
1143	* @param pVCpu The cross context virtual CPU structure.
1144	* @param pVmxTransient The VMX-transient structure.
1145	* @param uXcptMask The exception(s) to remove.
1146	*/
1147	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1148	{
1149	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1150	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1151	if (u32XcptBitmap & uXcptMask)
1152	{
1153	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1154	if (!pVmxTransient->fIsNestedGuest)
1155	{ /* likely */ }
1156	else
1157	{
1158	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1159	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1160	}
1161	#endif
1162	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1163	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1164	\| RT_BIT(X86_XCPT_DE)
1165	\| RT_BIT(X86_XCPT_NM)
1166	\| RT_BIT(X86_XCPT_TS)
1167	\| RT_BIT(X86_XCPT_UD)
1168	\| RT_BIT(X86_XCPT_NP)
1169	\| RT_BIT(X86_XCPT_SS)
1170	\| RT_BIT(X86_XCPT_GP)
1171	\| RT_BIT(X86_XCPT_PF)
1172	\| RT_BIT(X86_XCPT_MF));
1173	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1174	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1175	#endif
1176	if (uXcptMask)
1177	{
1178	/* Validate we are not removing any essential exception intercepts. */
1179	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1180	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1181	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1182
1183	/* Remove it from the exception bitmap. */
1184	u32XcptBitmap &= ~uXcptMask;
1185
1186	/* Commit and update the cache if necessary. */
1187	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1188	{
1189	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1190	AssertRCReturn(rc, rc);
1191	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1192	}
1193	}
1194	}
1195	return VINF_SUCCESS;
1196	}
1197
1198
1199	/**
1200	* Remove an exceptions from the exception bitmap and commits it to the current
1201	* VMCS.
1202	*
1203	* @returns VBox status code.
1204	* @param pVCpu The cross context virtual CPU structure.
1205	* @param pVmxTransient The VMX-transient structure.
1206	* @param uXcpt The exception to remove.
1207	*/
1208	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1209	{
1210	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1211	}
1212
1213
1214	/**
1215	* Loads the VMCS specified by the VMCS info. object.
1216	*
1217	* @returns VBox status code.
1218	* @param pVmcsInfo The VMCS info. object.
1219	*/
1220	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1221	{
1222	Assert(pVmcsInfo->HCPhysVmcs);
1223	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1224
1225	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1226	{
1227	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1228	if (RT_SUCCESS(rc))
1229	{
1230	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1231	return VINF_SUCCESS;
1232	}
1233	return rc;
1234	}
1235	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
1236	}
1237
1238
1239	/**
1240	* Clears the VMCS specified by the VMCS info. object.
1241	*
1242	* @returns VBox status code.
1243	* @param pVmcsInfo The VMCS info. object.
1244	*/
1245	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1246	{
1247	Assert(pVmcsInfo->HCPhysVmcs);
1248	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1249
1250	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1251	if (RT_SUCCESS(rc))
1252	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1253	return rc;
1254	}
1255
1256
1257	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1258	#if 0
1259	/**
1260	* Loads the shadow VMCS specified by the VMCS info. object.
1261	*
1262	* @returns VBox status code.
1263	* @param pVmcsInfo The VMCS info. object.
1264	*/
1265	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1266	{
1267	Assert(pVmcsInfo->HCPhysShadowVmcs);
1268	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1269
1270	if (pVmcsInfo->fShadowVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1271	{
1272	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1273	if (RT_SUCCESS(rc))
1274	{
1275	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_ACTIVE;
1276	return VINF_SUCCESS;
1277	}
1278	return rc;
1279	}
1280	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
1281	}
1282	#endif
1283
1284
1285	/**
1286	* Clears the shadow VMCS specified by the VMCS info. object.
1287	*
1288	* @returns VBox status code.
1289	* @param pVmcsInfo The VMCS info. object.
1290	*/
1291	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1292	{
1293	Assert(pVmcsInfo->HCPhysShadowVmcs);
1294	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1295
1296	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1297	if (RT_SUCCESS(rc))
1298	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1299	return rc;
1300	}
1301
1302
1303	/**
1304	* Switches the current VMCS to the one specified.
1305	*
1306	* @returns VBox status code.
1307	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1308	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1309	*
1310	* @remarks Called with interrupts disabled.
1311	*/
1312	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1313	{
1314	/*
1315	* Clear the VMCS we are switching out if it has not already been cleared.
1316	* This will sync any CPU internal data back to the VMCS.
1317	*/
1318	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1319	{
1320	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1321	if (RT_SUCCESS(rc))
1322	{ /* likely */ }
1323	else
1324	return rc;
1325	}
1326
1327	/*
1328	* Clear the VMCS we are switching to if it has not already been cleared.
1329	* This will initialize the VMCS launch state to "clear" required for loading it.
1330	*
1331	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1332	*/
1333	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1334	{
1335	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1336	if (RT_SUCCESS(rc))
1337	{ /* likely */ }
1338	else
1339	return rc;
1340	}
1341
1342	/*
1343	* Finally, load the VMCS we are switching to.
1344	*/
1345	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1346	}
1347	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1348
1349
1350	/**
1351	* Updates the VM's last error record.
1352	*
1353	* If there was a VMX instruction error, reads the error data from the VMCS and
1354	* updates VCPU's last error record as well.
1355	*
1356	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1357	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1358	* VERR_VMX_INVALID_VMCS_FIELD.
1359	* @param rc The error code.
1360	*/
1361	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1362	{
1363	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1364	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1365	{
1366	AssertPtrReturnVoid(pVCpu);
1367	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1368	}
1369	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1370	}
1371
1372
1373	#ifdef VBOX_STRICT
1374	/**
1375	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1376	* transient structure.
1377	*
1378	* @returns VBox status code.
1379	* @param pVmxTransient The VMX-transient structure.
1380	*
1381	* @remarks No-long-jump zone!!!
1382	*/
1383	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1384	{
1385	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1386	AssertRCReturn(rc, rc);
1387	return VINF_SUCCESS;
1388	}
1389
1390
1391	/**
1392	* Reads the VM-entry exception error code field from the VMCS into
1393	* the VMX transient structure.
1394	*
1395	* @returns VBox status code.
1396	* @param pVmxTransient The VMX-transient structure.
1397	*
1398	* @remarks No-long-jump zone!!!
1399	*/
1400	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1401	{
1402	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1403	AssertRCReturn(rc, rc);
1404	return VINF_SUCCESS;
1405	}
1406
1407
1408	/**
1409	* Reads the VM-entry exception error code field from the VMCS into
1410	* the VMX transient structure.
1411	*
1412	* @returns VBox status code.
1413	* @param pVmxTransient The VMX-transient structure.
1414	*
1415	* @remarks No-long-jump zone!!!
1416	*/
1417	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1418	{
1419	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1420	AssertRCReturn(rc, rc);
1421	return VINF_SUCCESS;
1422	}
1423	#endif /* VBOX_STRICT */
1424
1425
1426	/**
1427	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1428	* transient structure.
1429	*
1430	* @returns VBox status code.
1431	* @param pVmxTransient The VMX-transient structure.
1432	*/
1433	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1434	{
1435	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1436	{
1437	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1438	AssertRCReturn(rc,rc);
1439	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1440	}
1441	return VINF_SUCCESS;
1442	}
1443
1444
1445	/**
1446	* Reads the VM-exit interruption error code from the VMCS into the VMX
1447	* transient structure.
1448	*
1449	* @returns VBox status code.
1450	* @param pVmxTransient The VMX-transient structure.
1451	*/
1452	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1453	{
1454	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1455	{
1456	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1457	AssertRCReturn(rc, rc);
1458	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1459	}
1460	return VINF_SUCCESS;
1461	}
1462
1463
1464	/**
1465	* Reads the VM-exit instruction length field from the VMCS into the VMX
1466	* transient structure.
1467	*
1468	* @returns VBox status code.
1469	* @param pVmxTransient The VMX-transient structure.
1470	*/
1471	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1472	{
1473	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1474	{
1475	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1476	AssertRCReturn(rc, rc);
1477	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1478	}
1479	return VINF_SUCCESS;
1480	}
1481
1482
1483	/**
1484	* Reads the VM-exit instruction-information field from the VMCS into
1485	* the VMX transient structure.
1486	*
1487	* @returns VBox status code.
1488	* @param pVmxTransient The VMX-transient structure.
1489	*/
1490	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1491	{
1492	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1493	{
1494	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1495	AssertRCReturn(rc, rc);
1496	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1497	}
1498	return VINF_SUCCESS;
1499	}
1500
1501
1502	/**
1503	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1504	*
1505	* @returns VBox status code.
1506	* @param pVCpu The cross context virtual CPU structure of the
1507	* calling EMT. (Required for the VMCS cache case.)
1508	* @param pVmxTransient The VMX-transient structure.
1509	*/
1510	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1511	{
1512	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1513	{
1514	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1515	AssertRCReturn(rc, rc);
1516	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1517	}
1518	return VINF_SUCCESS;
1519	}
1520
1521
1522	/**
1523	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1524	*
1525	* @returns VBox status code.
1526	* @param pVCpu The cross context virtual CPU structure of the
1527	* calling EMT. (Required for the VMCS cache case.)
1528	* @param pVmxTransient The VMX-transient structure.
1529	*/
1530	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1531	{
1532	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1533	{
1534	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1535	AssertRCReturn(rc, rc);
1536	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1537	}
1538	return VINF_SUCCESS;
1539	}
1540
1541
1542	/**
1543	* Reads the IDT-vectoring information field from the VMCS into the VMX
1544	* transient structure.
1545	*
1546	* @returns VBox status code.
1547	* @param pVmxTransient The VMX-transient structure.
1548	*
1549	* @remarks No-long-jump zone!!!
1550	*/
1551	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1552	{
1553	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1554	{
1555	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1556	AssertRCReturn(rc, rc);
1557	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1558	}
1559	return VINF_SUCCESS;
1560	}
1561
1562
1563	/**
1564	* Reads the IDT-vectoring error code from the VMCS into the VMX
1565	* transient structure.
1566	*
1567	* @returns VBox status code.
1568	* @param pVmxTransient The VMX-transient structure.
1569	*/
1570	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1571	{
1572	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1573	{
1574	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1575	AssertRCReturn(rc, rc);
1576	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1577	}
1578	return VINF_SUCCESS;
1579	}
1580
1581
1582	/**
1583	* Enters VMX root mode operation on the current CPU.
1584	*
1585	* @returns VBox status code.
1586	* @param pVM The cross context VM structure. Can be
1587	* NULL, after a resume.
1588	* @param HCPhysCpuPage Physical address of the VMXON region.
1589	* @param pvCpuPage Pointer to the VMXON region.
1590	*/
1591	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1592	{
1593	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1594	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1595	Assert(pvCpuPage);
1596	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1597
1598	if (pVM)
1599	{
1600	/* Write the VMCS revision identifier to the VMXON region. */
1601	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1602	}
1603
1604	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1605	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1606
1607	/* Enable the VMX bit in CR4 if necessary. */
1608	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1609
1610	/* Enter VMX root mode. */
1611	int rc = VMXEnable(HCPhysCpuPage);
1612	if (RT_FAILURE(rc))
1613	{
1614	if (!(uOldCr4 & X86_CR4_VMXE))
1615	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1616
1617	if (pVM)
1618	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1619	}
1620
1621	/* Restore interrupts. */
1622	ASMSetFlags(fEFlags);
1623	return rc;
1624	}
1625
1626
1627	/**
1628	* Exits VMX root mode operation on the current CPU.
1629	*
1630	* @returns VBox status code.
1631	*/
1632	static int hmR0VmxLeaveRootMode(void)
1633	{
1634	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1635
1636	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1637	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1638
1639	/* If we're for some reason not in VMX root mode, then don't leave it. */
1640	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1641
1642	int rc;
1643	if (uHostCr4 & X86_CR4_VMXE)
1644	{
1645	/* Exit VMX root mode and clear the VMX bit in CR4. */
1646	VMXDisable();
1647	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1648	rc = VINF_SUCCESS;
1649	}
1650	else
1651	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1652
1653	/* Restore interrupts. */
1654	ASMSetFlags(fEFlags);
1655	return rc;
1656	}
1657
1658
1659	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1660	/**
1661	* Initializes the shadow VMCS.
1662	*
1663	* This builds an array (for use later while executing a nested-guest) of VMCS
1664	* fields to copy into the shadow VMCS.
1665	*
1666	* @param pVM The cross context VM structure.
1667	*/
1668	static void hmR0VmxInitShadowVmcsFieldsArray(PVM pVM)
1669	{
1670	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
1671	for (uint32_t i = 0; i < cVmcsFields; i++)
1672	{
1673	/*
1674	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
1675	* we must not include it in the shadow VMCS fields array. Guests attempting to
1676	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
1677	* the required behavior.
1678	*/
1679	uint32_t const uVmcsField = g_aVmcsFields[i];
1680	bool const fVmcsFieldValid = CPUMIsGuestVmxVmcsFieldValid(pVM, uVmcsField);
1681	if (fVmcsFieldValid)
1682	{
1683	pVM->hm.s.vmx.paShadowVmcsFields[i] = uVmcsField;
1684	++pVM->hm.s.vmx.cShadowVmcsFields;
1685	}
1686	}
1687	}
1688
1689
1690	/**
1691	* Initializes the VMREAD/VMWRITE bitmaps.
1692	*
1693	* @param pVM The cross context VM structure.
1694	*/
1695	static void hmR0VmxInitVmreadVmwriteBitmaps(PVM pVM)
1696	{
1697	/*
1698	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
1699	*/
1700	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
1701	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
1702	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
1703	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
1704	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
1705
1706	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
1707	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
1708
1709	/*
1710	* Initialize the VMREAD bitmap.
1711	* All valid guest VMCS fields (read-only and read-write) can be accessed
1712	* using VMREAD without causing a VM-exit.
1713	*/
1714	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1715	{
1716	uint32_t const uVmcsField = paShadowVmcsFields[i];
1717	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
1718	uint8_t *pbField = pbVmreadBitmap + (uVmcsField >> 3);
1719	ASMBitClear(pbField, uVmcsField & 7);
1720	}
1721
1722	/*
1723	* Initialize the VMWRITE bitmap.
1724	* Allow the guest to write to read-only guest VMCS fields only if the
1725	* host CPU supports it, otherwise it would cause a VMWRITE instruction error.
1726	*/
1727	bool const fHasVmwriteAll = RT_BOOL(pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL);
1728	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1729	{
1730	uint32_t const uVmcsField = paShadowVmcsFields[i];
1731	if ( fHasVmwriteAll
1732	\|\| !HMVmxIsVmcsFieldReadOnly(uVmcsField))
1733	{
1734	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
1735	uint8_t *pbField = pbVmwriteBitmap + (uVmcsField >> 3);
1736	ASMBitClear(pbField, uVmcsField & 7);
1737	}
1738	}
1739	}
1740	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1741
1742
1743	/**
1744	* Allocates and maps a physically contiguous page. The allocated page is
1745	* zero'd out (used by various VT-x structures).
1746	*
1747	* @returns IPRT status code.
1748	* @param pMemObj Pointer to the ring-0 memory object.
1749	* @param ppVirt Where to store the virtual address of the
1750	* allocation.
1751	* @param pHCPhys Where to store the physical address of the
1752	* allocation.
1753	*/
1754	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1755	{
1756	AssertPtr(pMemObj);
1757	AssertPtr(ppVirt);
1758	AssertPtr(pHCPhys);
1759	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1760	if (RT_FAILURE(rc))
1761	return rc;
1762	ppVirt = RTR0MemObjAddress(pMemObj);
1763	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1764	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1765	return VINF_SUCCESS;
1766	}
1767
1768
1769	/**
1770	* Frees and unmaps an allocated, physical page.
1771	*
1772	* @param pMemObj Pointer to the ring-0 memory object.
1773	* @param ppVirt Where to re-initialize the virtual address of
1774	* allocation as 0.
1775	* @param pHCPhys Where to re-initialize the physical address of the
1776	* allocation as 0.
1777	*/
1778	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1779	{
1780	AssertPtr(pMemObj);
1781	AssertPtr(ppVirt);
1782	AssertPtr(pHCPhys);
1783	/* NULL is valid, accepted and ignored by the free function below. */
1784	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1785	*pMemObj = NIL_RTR0MEMOBJ;
1786	*ppVirt = NULL;
1787	*pHCPhys = NIL_RTHCPHYS;
1788	}
1789
1790
1791	/**
1792	* Initializes a VMCS info. object.
1793	*
1794	* @param pVmcsInfo The VMCS info. object.
1795	*/
1796	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1797	{
1798	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1799
1800	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1801	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1802	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1803	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1804	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1805	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1806	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1807	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1808	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1809	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1810	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1811	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1812	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1813	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1814	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1815	}
1816
1817
1818	/**
1819	* Frees the VT-x structures for a VMCS info. object.
1820	*
1821	* @param pVM The cross context VM structure.
1822	* @param pVmcsInfo The VMCS info. object.
1823	*/
1824	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1825	{
1826	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1827
1828	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1829	if ( pVM->cpum.ro.GuestFeatures.fVmx
1830	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING))
1831	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1832	#endif
1833
1834	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1835	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1836
1837	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1838	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1839	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1840
1841	hmR0VmxInitVmcsInfo(pVmcsInfo);
1842	}
1843
1844
1845	/**
1846	* Allocates the VT-x structures for a VMCS info. object.
1847	*
1848	* @returns VBox status code.
1849	* @param pVCpu The cross context virtual CPU structure.
1850	* @param pVmcsInfo The VMCS info. object.
1851	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1852	*/
1853	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1854	{
1855	PVM pVM = pVCpu->CTX_SUFF(pVM);
1856
1857	/* Allocate the guest VM control structure (VMCS). */
1858	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1859	if (RT_SUCCESS(rc))
1860	{
1861	if (!fIsNstGstVmcs)
1862	{
1863	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1864	/* Allocate the shadow VMCS if supported by the CPU. */
1865	if ( pVM->cpum.ro.GuestFeatures.fVmx
1866	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING))
1867	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1868	#endif
1869	if (RT_SUCCESS(rc))
1870	{
1871	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1872	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1873	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1874	{
1875	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1876	NULL /* pR3Ptr /, NULL / pRCPtr */);
1877	}
1878	}
1879	}
1880	else
1881	{
1882	/* We don't yet support exposing VMCS shadowing to the guest. */
1883	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1884	Assert(!pVmcsInfo->pvShadowVmcs);
1885
1886	/* The host-physical address of the virtual-APIC page in guest memory is taken directly. */
1887	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1888	Assert(!pVmcsInfo->pbVirtApic);
1889	}
1890
1891	if (RT_SUCCESS(rc))
1892	{
1893	/*
1894	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1895	* transparent accesses of specific MSRs.
1896	*
1897	* If the condition for enabling MSR bitmaps changes here, don't forget to
1898	* update HMIsMsrBitmapActive().
1899	*
1900	* We don't share MSR bitmaps between the guest and nested-guest as we then
1901	* don't need to care about carefully restoring the guest MSR bitmap.
1902	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1903	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1904	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1905	* we do that later while merging VMCS.
1906	*/
1907	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1908	{
1909	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1910	if ( RT_SUCCESS(rc)
1911	&& !fIsNstGstVmcs)
1912	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1913	}
1914
1915	if (RT_SUCCESS(rc))
1916	{
1917	/*
1918	* Allocate the VM-entry MSR-load area for the guest MSRs.
1919	*
1920	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1921	* the guest and nested-guest.
1922	*/
1923	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1924	&pVmcsInfo->HCPhysGuestMsrLoad);
1925	if (RT_SUCCESS(rc))
1926	{
1927	/*
1928	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1929	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1930	*/
1931	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1932	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1933	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1934
1935	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1936	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1937	&pVmcsInfo->HCPhysHostMsrLoad);
1938	}
1939	}
1940	}
1941	}
1942
1943	return rc;
1944	}
1945
1946
1947	/**
1948	* Free all VT-x structures for the VM.
1949	*
1950	* @returns IPRT status code.
1951	* @param pVM The cross context VM structure.
1952	*/
1953	static void hmR0VmxStructsFree(PVM pVM)
1954	{
1955	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1956	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1957	#endif
1958	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1959
1960	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1961	if ( pVM->cpum.ro.GuestFeatures.fVmx
1962	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING))
1963	{
1964	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1965	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1966	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1967	}
1968	#endif
1969
1970	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1971	{
1972	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1973	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1974	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1975	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1976	if (pVM->cpum.ro.GuestFeatures.fVmx)
1977	{
1978	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1979	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1980	}
1981	#endif
1982	}
1983	}
1984
1985
1986	/**
1987	* Allocate all VT-x structures for the VM.
1988	*
1989	* @returns IPRT status code.
1990	* @param pVM The cross context VM structure.
1991	*/
1992	static int hmR0VmxStructsAlloc(PVM pVM)
1993	{
1994	/*
1995	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1996	* The VMCS size cannot be more than 4096 bytes.
1997	*
1998	* See Intel spec. Appendix A.1 "Basic VMX Information".
1999	*/
2000	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
2001	if (cbVmcs <= X86_PAGE_4K_SIZE)
2002	{ /* likely */ }
2003	else
2004	{
2005	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
2006	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2007	}
2008
2009	/*
2010	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
2011	*/
2012	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2013	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
2014	Assert(pVM->hm.s.vmx.pbScratch == NULL);
2015	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
2016	#endif
2017
2018	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
2019	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
2020	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
2021
2022	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2023	{
2024	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2025	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
2026	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
2027	}
2028
2029	/*
2030	* Allocate per-VM VT-x structures.
2031	*/
2032	int rc = VINF_SUCCESS;
2033	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2034	/* Allocate crash-dump magic scratch page. */
2035	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
2036	if (RT_FAILURE(rc))
2037	{
2038	hmR0VmxStructsFree(pVM);
2039	return rc;
2040	}
2041	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
2042	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
2043	#endif
2044
2045	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
2046	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2047	{
2048	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
2049	&pVM->hm.s.vmx.HCPhysApicAccess);
2050	if (RT_FAILURE(rc))
2051	{
2052	hmR0VmxStructsFree(pVM);
2053	return rc;
2054	}
2055	}
2056
2057	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2058	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps if VMCS shadowing supported by the CPU. */
2059	if ( pVM->cpum.ro.GuestFeatures.fVmx
2060	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING))
2061	{
2062	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
2063	if (RT_LIKELY(pVM->hm.s.vmx.paShadowVmcsFields))
2064	{
2065	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
2066	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
2067	if (RT_SUCCESS(rc))
2068	{
2069	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
2070	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
2071	if (RT_SUCCESS(rc))
2072	{
2073	hmR0VmxInitShadowVmcsFieldsArray(pVM);
2074	hmR0VmxInitVmreadVmwriteBitmaps(pVM);
2075	}
2076	}
2077	}
2078	else
2079	rc = VERR_NO_MEMORY;
2080
2081	if (RT_FAILURE(rc))
2082	{
2083	hmR0VmxStructsFree(pVM);
2084	return rc;
2085	}
2086	}
2087	#endif
2088
2089	/*
2090	* Initialize per-VCPU VT-x structures.
2091	*/
2092	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2093	{
2094	/* Allocate the guest VMCS structures. */
2095	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2096	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2097	if (RT_SUCCESS(rc))
2098	{
2099	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2100	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2101	if (pVM->cpum.ro.GuestFeatures.fVmx)
2102	{
2103	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2104	if (RT_SUCCESS(rc))
2105	{ /* likely */ }
2106	else
2107	break;
2108	}
2109	#endif
2110	}
2111	else
2112	break;
2113	}
2114
2115	if (RT_FAILURE(rc))
2116	{
2117	hmR0VmxStructsFree(pVM);
2118	return rc;
2119	}
2120
2121	return VINF_SUCCESS;
2122	}
2123
2124	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2125	/**
2126	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2127	*
2128	* @returns @c true if the MSR is intercepted, @c false otherwise.
2129	* @param pvMsrBitmap The MSR bitmap.
2130	* @param offMsr The MSR byte offset.
2131	* @param iBit The bit offset from the byte offset.
2132	*/
2133	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2134	{
2135	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2136	Assert(pbMsrBitmap);
2137	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2138	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2139	}
2140	#endif
2141
2142	/**
2143	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2144	*
2145	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2146	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2147	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2148	* the read/write access of this MSR.
2149	*
2150	* @param pVCpu The cross context virtual CPU structure.
2151	* @param pVmcsInfo The VMCS info. object.
2152	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2153	* @param idMsr The MSR value.
2154	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2155	* include both a read -and- a write permission!
2156	*
2157	* @sa CPUMGetVmxMsrPermission.
2158	* @remarks Can be called with interrupts disabled.
2159	*/
2160	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2161	{
2162	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2163	Assert(pbMsrBitmap);
2164	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2165
2166	/*
2167	* MSR-bitmap Layout:
2168	* Byte index MSR range Interpreted as
2169	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2170	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2171	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2172	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2173	*
2174	* A bit corresponding to an MSR within the above range causes a VM-exit
2175	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2176	* the MSR range, it always cause a VM-exit.
2177	*
2178	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2179	*/
2180	uint16_t const offBitmapRead = 0;
2181	uint16_t const offBitmapWrite = 0x800;
2182	uint16_t offMsr;
2183	int32_t iBit;
2184	if (idMsr <= UINT32_C(0x00001fff))
2185	{
2186	offMsr = 0;
2187	iBit = idMsr;
2188	}
2189	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2190	{
2191	offMsr = 0x400;
2192	iBit = idMsr - UINT32_C(0xc0000000);
2193	}
2194	else
2195	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2196
2197	/*
2198	* Set the MSR read permission.
2199	*/
2200	uint16_t const offMsrRead = offBitmapRead + offMsr;
2201	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2202	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2203	{
2204	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2205	bool const fClear = !fIsNstGstVmcs ? true
2206	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2207	#else
2208	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2209	bool const fClear = true;
2210	#endif
2211	if (fClear)
2212	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2213	}
2214	else
2215	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2216
2217	/*
2218	* Set the MSR write permission.
2219	*/
2220	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2221	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2222	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2223	{
2224	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2225	bool const fClear = !fIsNstGstVmcs ? true
2226	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2227	#else
2228	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2229	bool const fClear = true;
2230	#endif
2231	if (fClear)
2232	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2233	}
2234	else
2235	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2236	}
2237
2238
2239	/**
2240	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2241	* area.
2242	*
2243	* @returns VBox status code.
2244	* @param pVCpu The cross context virtual CPU structure.
2245	* @param pVmcsInfo The VMCS info. object.
2246	* @param cMsrs The number of MSRs.
2247	*/
2248	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2249	{
2250	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2251	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2252	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2253	{
2254	/* Commit the MSR counts to the VMCS and update the cache. */
2255	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2256	{
2257	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2258	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2259	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2260	AssertRCReturn(rc, rc);
2261
2262	pVmcsInfo->cEntryMsrLoad = cMsrs;
2263	pVmcsInfo->cExitMsrStore = cMsrs;
2264	pVmcsInfo->cExitMsrLoad = cMsrs;
2265	}
2266	return VINF_SUCCESS;
2267	}
2268
2269	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2270	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2271	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272	}
2273
2274
2275	/**
2276	* Adds a new (or updates the value of an existing) guest/host MSR
2277	* pair to be swapped during the world-switch as part of the
2278	* auto-load/store MSR area in the VMCS.
2279	*
2280	* @returns VBox status code.
2281	* @param pVCpu The cross context virtual CPU structure.
2282	* @param pVmxTransient The VMX-transient structure.
2283	* @param idMsr The MSR.
2284	* @param uGuestMsrValue Value of the guest MSR.
2285	* @param fSetReadWrite Whether to set the guest read/write access of this
2286	* MSR (thus not causing a VM-exit).
2287	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2288	* necessary.
2289	*/
2290	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2291	bool fSetReadWrite, bool fUpdateHostMsr)
2292	{
2293	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2294	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2295	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2296	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2297	uint32_t i;
2298
2299	/* Paranoia. */
2300	Assert(pGuestMsrLoad);
2301
2302	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2303
2304	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2305	for (i = 0; i < cMsrs; i++)
2306	{
2307	if (pGuestMsrLoad[i].u32Msr == idMsr)
2308	break;
2309	}
2310
2311	bool fAdded = false;
2312	if (i == cMsrs)
2313	{
2314	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2315	++cMsrs;
2316	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2317	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2318
2319	/* Set the guest to read/write this MSR without causing VM-exits. */
2320	if ( fSetReadWrite
2321	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2322	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2323
2324	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2325	fAdded = true;
2326	}
2327
2328	/* Update the MSR value for the newly added or already existing MSR. */
2329	pGuestMsrLoad[i].u32Msr = idMsr;
2330	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2331
2332	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2333	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2334	{
2335	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2336	pGuestMsrStore[i].u32Msr = idMsr;
2337	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2338	}
2339
2340	/* Update the corresponding slot in the host MSR area. */
2341	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2342	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2343	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2344	pHostMsr[i].u32Msr = idMsr;
2345
2346	/*
2347	* Only if the caller requests to update the host MSR value AND we've newly added the
2348	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2349	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2350	*
2351	* We do this for performance reasons since reading MSRs may be quite expensive.
2352	*/
2353	if (fAdded)
2354	{
2355	if (fUpdateHostMsr)
2356	{
2357	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2358	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2359	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2360	}
2361	else
2362	{
2363	/* Someone else can do the work. */
2364	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2365	}
2366	}
2367	return VINF_SUCCESS;
2368	}
2369
2370
2371	/**
2372	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2373	* auto-load/store MSR area in the VMCS.
2374	*
2375	* @returns VBox status code.
2376	* @param pVCpu The cross context virtual CPU structure.
2377	* @param pVmxTransient The VMX-transient structure.
2378	* @param idMsr The MSR.
2379	*/
2380	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2381	{
2382	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2383	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2384	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2385	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2386
2387	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2388
2389	for (uint32_t i = 0; i < cMsrs; i++)
2390	{
2391	/* Find the MSR. */
2392	if (pGuestMsrLoad[i].u32Msr == idMsr)
2393	{
2394	/*
2395	* If it's the last MSR, we only need to reduce the MSR count.
2396	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2397	*/
2398	if (i < cMsrs - 1)
2399	{
2400	/* Remove it from the VM-entry MSR-load area. */
2401	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2402	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2403
2404	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2405	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2406	{
2407	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2408	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2409	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2410	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2411	}
2412
2413	/* Remove it from the VM-exit MSR-load area. */
2414	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2415	Assert(pHostMsr[i].u32Msr == idMsr);
2416	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2417	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2418	}
2419
2420	/* Reduce the count to reflect the removed MSR and bail. */
2421	--cMsrs;
2422	break;
2423	}
2424	}
2425
2426	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2427	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2428	{
2429	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2430	AssertRCReturn(rc, rc);
2431
2432	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2433	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2434	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2435
2436	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2437	return VINF_SUCCESS;
2438	}
2439
2440	return VERR_NOT_FOUND;
2441	}
2442
2443
2444	/**
2445	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2446	*
2447	* @returns @c true if found, @c false otherwise.
2448	* @param pVmcsInfo The VMCS info. object.
2449	* @param idMsr The MSR to find.
2450	*/
2451	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2452	{
2453	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2454	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2455	Assert(pMsrs);
2456	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2457	for (uint32_t i = 0; i < cMsrs; i++)
2458	{
2459	if (pMsrs[i].u32Msr == idMsr)
2460	return true;
2461	}
2462	return false;
2463	}
2464
2465
2466	/**
2467	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2468	*
2469	* @param pVCpu The cross context virtual CPU structure.
2470	* @param pVmcsInfo The VMCS info. object.
2471	*
2472	* @remarks No-long-jump zone!!!
2473	*/
2474	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2475	{
2476	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2477
2478	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2479	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2480	Assert(pHostMsrLoad);
2481	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2482	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2483	for (uint32_t i = 0; i < cMsrs; i++)
2484	{
2485	/*
2486	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2487	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2488	*/
2489	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2490	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2491	else
2492	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2493	}
2494	}
2495
2496
2497	/**
2498	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2499	* perform lazy restoration of the host MSRs while leaving VT-x.
2500	*
2501	* @param pVCpu The cross context virtual CPU structure.
2502	*
2503	* @remarks No-long-jump zone!!!
2504	*/
2505	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2506	{
2507	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2508
2509	/*
2510	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2511	*/
2512	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2513	{
2514	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2515	#if HC_ARCH_BITS == 64
2516	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2517	{
2518	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2519	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2520	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2521	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2522	}
2523	#endif
2524	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2525	}
2526	}
2527
2528
2529	/**
2530	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2531	* lazily while leaving VT-x.
2532	*
2533	* @returns true if it does, false otherwise.
2534	* @param pVCpu The cross context virtual CPU structure.
2535	* @param idMsr The MSR to check.
2536	*/
2537	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2538	{
2539	NOREF(pVCpu);
2540	#if HC_ARCH_BITS == 64
2541	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2542	{
2543	switch (idMsr)
2544	{
2545	case MSR_K8_LSTAR:
2546	case MSR_K6_STAR:
2547	case MSR_K8_SF_MASK:
2548	case MSR_K8_KERNEL_GS_BASE:
2549	return true;
2550	}
2551	}
2552	#else
2553	RT_NOREF(pVCpu, idMsr);
2554	#endif
2555	return false;
2556	}
2557
2558
2559	/**
2560	* Loads a set of guests MSRs to allow read/passthru to the guest.
2561	*
2562	* The name of this function is slightly confusing. This function does NOT
2563	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2564	* common prefix for functions dealing with "lazy restoration" of the shared
2565	* MSRs.
2566	*
2567	* @param pVCpu The cross context virtual CPU structure.
2568	*
2569	* @remarks No-long-jump zone!!!
2570	*/
2571	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2572	{
2573	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2574	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2575
2576	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2577	#if HC_ARCH_BITS == 64
2578	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2579	{
2580	/*
2581	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2582	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2583	* we can skip a few MSR writes.
2584	*
2585	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2586	* guest MSR values in the guest-CPU context might be different to what's currently
2587	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2588	* CPU, see @bugref{8728}.
2589	*/
2590	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2591	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2592	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2593	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2594	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2595	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2596	{
2597	#ifdef VBOX_STRICT
2598	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2599	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2600	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2601	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2602	#endif
2603	}
2604	else
2605	{
2606	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2607	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2608	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2609	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2610	}
2611	}
2612	#endif
2613	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2614	}
2615
2616
2617	/**
2618	* Performs lazy restoration of the set of host MSRs if they were previously
2619	* loaded with guest MSR values.
2620	*
2621	* @param pVCpu The cross context virtual CPU structure.
2622	*
2623	* @remarks No-long-jump zone!!!
2624	* @remarks The guest MSRs should have been saved back into the guest-CPU
2625	* context by hmR0VmxImportGuestState()!!!
2626	*/
2627	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2628	{
2629	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2630	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2631
2632	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2633	{
2634	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2635	#if HC_ARCH_BITS == 64
2636	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2637	{
2638	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2639	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2640	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2641	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2642	}
2643	#endif
2644	}
2645	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2646	}
2647
2648
2649	/**
2650	* Verifies that our cached values of the VMCS fields are all consistent with
2651	* what's actually present in the VMCS.
2652	*
2653	* @returns VBox status code.
2654	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2655	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2656	* VMCS content. HMCPU error-field is
2657	* updated, see VMX_VCI_XXX.
2658	* @param pVCpu The cross context virtual CPU structure.
2659	* @param pVmcsInfo The VMCS info. object.
2660	*/
2661	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2662	{
2663	uint32_t u32Val;
2664	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2665	AssertRCReturn(rc, rc);
2666	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2667	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2668	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2669	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2670
2671	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2672	AssertRCReturn(rc, rc);
2673	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2674	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2675	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2676	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2677
2678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2679	AssertRCReturn(rc, rc);
2680	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2681	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2682	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2683	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2684
2685	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2686	AssertRCReturn(rc, rc);
2687	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2688	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2689	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2690	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2691
2692	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2693	{
2694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2695	AssertRCReturn(rc, rc);
2696	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2697	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2698	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2699	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2700	}
2701
2702	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2703	AssertRCReturn(rc, rc);
2704	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2705	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2706	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2707	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2708
2709	uint64_t u64Val;
2710	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2711	AssertRCReturn(rc, rc);
2712	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2713	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2714	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2715	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2716
2717	return VINF_SUCCESS;
2718	}
2719
2720
2721	#ifdef VBOX_STRICT
2722	/**
2723	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2724	*
2725	* @param pVCpu The cross context virtual CPU structure.
2726	* @param pVmcsInfo The VMCS info. object.
2727	*/
2728	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2729	{
2730	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2731
2732	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2733	{
2734	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2735	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2736	uint64_t uVmcsEferMsrVmcs;
2737	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2738	AssertRC(rc);
2739
2740	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2741	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2742	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2743	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2744	}
2745	}
2746
2747
2748	/**
2749	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2750	* VMCS are correct.
2751	*
2752	* @param pVCpu The cross context virtual CPU structure.
2753	* @param pVmcsInfo The VMCS info. object.
2754	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2755	*/
2756	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2757	{
2758	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2759
2760	/* Read the various MSR-area counts from the VMCS. */
2761	uint32_t cEntryLoadMsrs;
2762	uint32_t cExitStoreMsrs;
2763	uint32_t cExitLoadMsrs;
2764	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2765	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2766	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2767
2768	/* Verify all the MSR counts are the same. */
2769	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2770	Assert(cExitStoreMsrs == cExitLoadMsrs);
2771	uint32_t const cMsrs = cExitLoadMsrs;
2772
2773	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2774	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2775
2776	/* Verify the MSR counts are within the allocated page size. */
2777	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2778
2779	/* Verify the relevant contents of the MSR areas match. */
2780	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2781	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2782	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2783	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2784	for (uint32_t i = 0; i < cMsrs; i++)
2785	{
2786	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2787	if (fSeparateExitMsrStorePage)
2788	{
2789	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2790	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2791	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2792	}
2793
2794	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2795	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2796	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2797
2798	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2799	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2800	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2801	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2802
2803	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2804	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2805	if (fIsEferMsr)
2806	{
2807	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2808	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2809	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2810	}
2811
2812	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2813	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2814	{
2815	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2816	if (fIsEferMsr)
2817	{
2818	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2819	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2820	}
2821	else
2822	{
2823	if (!fIsNstGstVmcs)
2824	{
2825	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2826	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2827	}
2828	else
2829	{
2830	/*
2831	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2832	* execute a nested-guest with MSR passthrough.
2833	*
2834	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2835	* allow passthrough too.
2836	*/
2837	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2838	Assert(pvMsrBitmapNstGst);
2839	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2840	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2841	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2842	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2843	}
2844	}
2845	}
2846
2847	/* Move to the next MSR. */
2848	pHostMsrLoad++;
2849	pGuestMsrLoad++;
2850	pGuestMsrStore++;
2851	}
2852	}
2853	#endif /* VBOX_STRICT */
2854
2855
2856	/**
2857	* Flushes the TLB using EPT.
2858	*
2859	* @returns VBox status code.
2860	* @param pVCpu The cross context virtual CPU structure of the calling
2861	* EMT. Can be NULL depending on @a enmTlbFlush.
2862	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2863	* enmTlbFlush.
2864	* @param enmTlbFlush Type of flush.
2865	*
2866	* @remarks Caller is responsible for making sure this function is called only
2867	* when NestedPaging is supported and providing @a enmTlbFlush that is
2868	* supported by the CPU.
2869	* @remarks Can be called with interrupts disabled.
2870	*/
2871	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2872	{
2873	uint64_t au64Descriptor[2];
2874	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2875	au64Descriptor[0] = 0;
2876	else
2877	{
2878	Assert(pVCpu);
2879	Assert(pVmcsInfo);
2880	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2881	}
2882	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2883
2884	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2885	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2886
2887	if ( RT_SUCCESS(rc)
2888	&& pVCpu)
2889	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2890	}
2891
2892
2893	/**
2894	* Flushes the TLB using VPID.
2895	*
2896	* @returns VBox status code.
2897	* @param pVCpu The cross context virtual CPU structure of the calling
2898	* EMT. Can be NULL depending on @a enmTlbFlush.
2899	* @param enmTlbFlush Type of flush.
2900	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2901	* on @a enmTlbFlush).
2902	*
2903	* @remarks Can be called with interrupts disabled.
2904	*/
2905	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2906	{
2907	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2908
2909	uint64_t au64Descriptor[2];
2910	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2911	{
2912	au64Descriptor[0] = 0;
2913	au64Descriptor[1] = 0;
2914	}
2915	else
2916	{
2917	AssertPtr(pVCpu);
2918	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2919	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2920	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2921	au64Descriptor[1] = GCPtr;
2922	}
2923
2924	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2925	AssertMsg(rc == VINF_SUCCESS,
2926	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2927
2928	if ( RT_SUCCESS(rc)
2929	&& pVCpu)
2930	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2931	NOREF(rc);
2932	}
2933
2934
2935	/**
2936	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2937	* otherwise there is nothing really to invalidate.
2938	*
2939	* @returns VBox status code.
2940	* @param pVCpu The cross context virtual CPU structure.
2941	* @param GCVirt Guest virtual address of the page to invalidate.
2942	*/
2943	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2944	{
2945	AssertPtr(pVCpu);
2946	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2947
2948	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2949	{
2950	/*
2951	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2952	* the EPT case. See @bugref{6043} and @bugref{6177}.
2953	*
2954	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2955	* as this function maybe called in a loop with individual addresses.
2956	*/
2957	PVM pVM = pVCpu->CTX_SUFF(pVM);
2958	if (pVM->hm.s.vmx.fVpid)
2959	{
2960	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2961
2962	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2963	/*
2964	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2965	* where executing INVVPID outside 64-bit mode does not flush translations of
2966	* 64-bit linear addresses, see @bugref{6208#c72}.
2967	*/
2968	if (RT_HI_U32(GCVirt))
2969	fVpidFlush = false;
2970	#endif
2971
2972	if (fVpidFlush)
2973	{
2974	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2975	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2976	}
2977	else
2978	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2979	}
2980	else if (pVM->hm.s.fNestedPaging)
2981	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2982	}
2983
2984	return VINF_SUCCESS;
2985	}
2986
2987
2988	/**
2989	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2990	* case where neither EPT nor VPID is supported by the CPU.
2991	*
2992	* @param pHostCpu The HM physical-CPU structure.
2993	* @param pVCpu The cross context virtual CPU structure.
2994	*
2995	* @remarks Called with interrupts disabled.
2996	*/
2997	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2998	{
2999	AssertPtr(pVCpu);
3000	AssertPtr(pHostCpu);
3001
3002	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
3003
3004	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3005	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3006	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3007	pVCpu->hm.s.fForceTLBFlush = false;
3008	return;
3009	}
3010
3011
3012	/**
3013	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
3014	*
3015	* @param pHostCpu The HM physical-CPU structure.
3016	* @param pVCpu The cross context virtual CPU structure.
3017	* @param pVmcsInfo The VMCS info. object.
3018	*
3019	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
3020	* nomenclature. The reason is, to avoid confusion in compare statements
3021	* since the host-CPU copies are named "ASID".
3022	*
3023	* @remarks Called with interrupts disabled.
3024	*/
3025	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3026	{
3027	#ifdef VBOX_WITH_STATISTICS
3028	bool fTlbFlushed = false;
3029	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
3030	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
3031	if (!fTlbFlushed) \
3032	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
3033	} while (0)
3034	#else
3035	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
3036	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
3037	#endif
3038
3039	AssertPtr(pVCpu);
3040	AssertPtr(pHostCpu);
3041	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3042
3043	PVM pVM = pVCpu->CTX_SUFF(pVM);
3044	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
3045	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
3046	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
3047
3048	/*
3049	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
3050	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3051	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3052	* cannot reuse the current ASID anymore.
3053	*/
3054	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3055	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3056	{
3057	++pHostCpu->uCurrentAsid;
3058	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3059	{
3060	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
3061	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3062	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3063	}
3064
3065	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3066	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3067	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3068
3069	/*
3070	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
3071	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
3072	*/
3073	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3074	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3075	HMVMX_SET_TAGGED_TLB_FLUSHED();
3076	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
3077	}
3078	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
3079	{
3080	/*
3081	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
3082	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
3083	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
3084	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
3085	* mappings, see @bugref{6568}.
3086	*
3087	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
3088	*/
3089	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3090	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3091	HMVMX_SET_TAGGED_TLB_FLUSHED();
3092	}
3093
3094	pVCpu->hm.s.fForceTLBFlush = false;
3095	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3096
3097	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3098	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3099	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3100	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3101	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3102	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3103	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3104	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3105	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3106
3107	/* Update VMCS with the VPID. */
3108	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3109	AssertRC(rc);
3110
3111	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3112	}
3113
3114
3115	/**
3116	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3117	*
3118	* @param pHostCpu The HM physical-CPU structure.
3119	* @param pVCpu The cross context virtual CPU structure.
3120	* @param pVmcsInfo The VMCS info. object.
3121	*
3122	* @remarks Called with interrupts disabled.
3123	*/
3124	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3125	{
3126	AssertPtr(pVCpu);
3127	AssertPtr(pHostCpu);
3128	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3129	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3130	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3131
3132	/*
3133	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3134	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3135	*/
3136	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3137	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3138	{
3139	pVCpu->hm.s.fForceTLBFlush = true;
3140	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3141	}
3142
3143	/* Check for explicit TLB flushes. */
3144	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3145	{
3146	pVCpu->hm.s.fForceTLBFlush = true;
3147	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3148	}
3149
3150	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3151	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3152
3153	if (pVCpu->hm.s.fForceTLBFlush)
3154	{
3155	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3156	pVCpu->hm.s.fForceTLBFlush = false;
3157	}
3158	}
3159
3160
3161	/**
3162	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3163	*
3164	* @param pHostCpu The HM physical-CPU structure.
3165	* @param pVCpu The cross context virtual CPU structure.
3166	*
3167	* @remarks Called with interrupts disabled.
3168	*/
3169	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3170	{
3171	AssertPtr(pVCpu);
3172	AssertPtr(pHostCpu);
3173	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3174	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3175	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3176
3177	/*
3178	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3179	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3180	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3181	* cannot reuse the current ASID anymore.
3182	*/
3183	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3184	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3185	{
3186	pVCpu->hm.s.fForceTLBFlush = true;
3187	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3188	}
3189
3190	/* Check for explicit TLB flushes. */
3191	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3192	{
3193	/*
3194	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3195	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3196	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3197	* include fExplicitFlush's too) - an obscure corner case.
3198	*/
3199	pVCpu->hm.s.fForceTLBFlush = true;
3200	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3201	}
3202
3203	PVM pVM = pVCpu->CTX_SUFF(pVM);
3204	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3205	if (pVCpu->hm.s.fForceTLBFlush)
3206	{
3207	++pHostCpu->uCurrentAsid;
3208	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3209	{
3210	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3211	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3212	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3213	}
3214
3215	pVCpu->hm.s.fForceTLBFlush = false;
3216	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3217	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3218	if (pHostCpu->fFlushAsidBeforeUse)
3219	{
3220	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3221	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3222	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3223	{
3224	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3225	pHostCpu->fFlushAsidBeforeUse = false;
3226	}
3227	else
3228	{
3229	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3230	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3231	}
3232	}
3233	}
3234
3235	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3236	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3237	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3238	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3239	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3240	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3241	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3242
3243	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3244	AssertRC(rc);
3245	}
3246
3247
3248	/**
3249	* Flushes the guest TLB entry based on CPU capabilities.
3250	*
3251	* @param pHostCpu The HM physical-CPU structure.
3252	* @param pVCpu The cross context virtual CPU structure.
3253	* @param pVmcsInfo The VMCS info. object.
3254	*
3255	* @remarks Called with interrupts disabled.
3256	*/
3257	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3258	{
3259	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3260	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3261	#endif
3262	PVM pVM = pVCpu->CTX_SUFF(pVM);
3263	switch (pVM->hm.s.vmx.enmTlbFlushType)
3264	{
3265	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3266	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3267	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3268	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3269	default:
3270	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3271	break;
3272	}
3273	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3274	}
3275
3276
3277	/**
3278	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3279	* TLB entries from the host TLB before VM-entry.
3280	*
3281	* @returns VBox status code.
3282	* @param pVM The cross context VM structure.
3283	*/
3284	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3285	{
3286	/*
3287	* Determine optimal flush type for nested paging.
3288	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3289	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3290	*/
3291	if (pVM->hm.s.fNestedPaging)
3292	{
3293	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3294	{
3295	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3296	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3297	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3298	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3299	else
3300	{
3301	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3302	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3303	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3304	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3305	}
3306
3307	/* Make sure the write-back cacheable memory type for EPT is supported. */
3308	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3309	{
3310	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3311	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3312	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3313	}
3314
3315	/* EPT requires a page-walk length of 4. */
3316	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3317	{
3318	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3319	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3320	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3321	}
3322	}
3323	else
3324	{
3325	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3326	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3327	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3328	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3329	}
3330	}
3331
3332	/*
3333	* Determine optimal flush type for VPID.
3334	*/
3335	if (pVM->hm.s.vmx.fVpid)
3336	{
3337	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3338	{
3339	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3340	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3341	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3342	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3343	else
3344	{
3345	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3346	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3347	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3348	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3349	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3350	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3351	pVM->hm.s.vmx.fVpid = false;
3352	}
3353	}
3354	else
3355	{
3356	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3357	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3358	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3359	pVM->hm.s.vmx.fVpid = false;
3360	}
3361	}
3362
3363	/*
3364	* Setup the handler for flushing tagged-TLBs.
3365	*/
3366	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3367	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3368	else if (pVM->hm.s.fNestedPaging)
3369	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3370	else if (pVM->hm.s.vmx.fVpid)
3371	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3372	else
3373	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3374	return VINF_SUCCESS;
3375	}
3376
3377
3378	/**
3379	* Sets up the virtual-APIC page address for the VMCS.
3380	*
3381	* @returns VBox status code.
3382	* @param pVCpu The cross context virtual CPU structure.
3383	* @param pVmcsInfo The VMCS info. object.
3384	*/
3385	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3386	{
3387	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3388	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3389	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3390	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3391	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3392	}
3393
3394
3395	/**
3396	* Sets up the MSR-bitmap address for the VMCS.
3397	*
3398	* @returns VBox status code.
3399	* @param pVCpu The cross context virtual CPU structure.
3400	* @param pVmcsInfo The VMCS info. object.
3401	*/
3402	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3403	{
3404	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3405	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3406	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3407	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3408	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3409	}
3410
3411
3412	/**
3413	* Sets up the APIC-access page address for the VMCS.
3414	*
3415	* @returns VBox status code.
3416	* @param pVCpu The cross context virtual CPU structure.
3417	*/
3418	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3419	{
3420	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3421	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3422	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3423	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3424	}
3425
3426
3427	/**
3428	* Sets up the VMCS link pointer for the VMCS.
3429	*
3430	* @returns VBox status code.
3431	* @param pVCpu The cross context virtual CPU structure.
3432	* @param pVmcsInfo The VMCS info. object.
3433	*/
3434	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3435	{
3436	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3437	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
3438	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
3439	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
3440	}
3441
3442
3443	/**
3444	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3445	* in the VMCS.
3446	*
3447	* @returns VBox status code.
3448	* @param pVCpu The cross context virtual CPU structure.
3449	* @param pVmcsInfo The VMCS info. object.
3450	*/
3451	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3452	{
3453	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3454
3455	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3456	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3457	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3458
3459	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3460	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3461	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3462
3463	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3464	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3465	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3466
3467	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3468	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3469	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3470	AssertRCReturn(rc, rc);
3471	return VINF_SUCCESS;
3472	}
3473
3474
3475	/**
3476	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3477	*
3478	* @param pVCpu The cross context virtual CPU structure.
3479	* @param pVmcsInfo The VMCS info. object.
3480	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3481	*/
3482	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3483	{
3484	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3485
3486	/*
3487	* The guest can access the following MSRs (read, write) without causing
3488	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3489	*/
3490	PVM pVM = pVCpu->CTX_SUFF(pVM);
3491	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3492	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3493	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3494	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3495	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3496
3497	/*
3498	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3499	* associated with then. We never need to intercept access (writes need to be
3500	* executed without causing a VM-exit, reads will #GP fault anyway).
3501	*
3502	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3503	* read/write them. We swap the the guest/host MSR value using the
3504	* auto-load/store MSR area.
3505	*/
3506	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3507	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3508	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3509	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3510	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3511	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3512
3513	#if HC_ARCH_BITS == 64
3514	/*
3515	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3516	* required for 64-bit guests.
3517	*/
3518	if (pVM->hm.s.fAllow64BitGuests)
3519	{
3520	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3521	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3522	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3523	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3524	}
3525	#endif
3526
3527	/*
3528	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3529	*/
3530	#ifdef VBOX_STRICT
3531	Assert(pVmcsInfo->pvMsrBitmap);
3532	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3533	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3534	#endif
3535	}
3536
3537
3538	/**
3539	* Sets up pin-based VM-execution controls in the VMCS.
3540	*
3541	* @returns VBox status code.
3542	* @param pVCpu The cross context virtual CPU structure.
3543	* @param pVmcsInfo The VMCS info. object.
3544	*/
3545	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3546	{
3547	PVM pVM = pVCpu->CTX_SUFF(pVM);
3548	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3549	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3550
3551	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3552	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3553
3554	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3555	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3556
3557	/* Enable the VMX-preemption timer. */
3558	if (pVM->hm.s.vmx.fUsePreemptTimer)
3559	{
3560	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3561	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3562	}
3563
3564	#if 0
3565	/* Enable posted-interrupt processing. */
3566	if (pVM->hm.s.fPostedIntrs)
3567	{
3568	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3569	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3570	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3571	}
3572	#endif
3573
3574	if ((fVal & fZap) != fVal)
3575	{
3576	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3577	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3578	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3579	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3580	}
3581
3582	/* Commit it to the VMCS and update our cache. */
3583	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3584	AssertRCReturn(rc, rc);
3585	pVmcsInfo->u32PinCtls = fVal;
3586
3587	return VINF_SUCCESS;
3588	}
3589
3590
3591	/**
3592	* Sets up secondary processor-based VM-execution controls in the VMCS.
3593	*
3594	* @returns VBox status code.
3595	* @param pVCpu The cross context virtual CPU structure.
3596	* @param pVmcsInfo The VMCS info. object.
3597	*/
3598	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3599	{
3600	PVM pVM = pVCpu->CTX_SUFF(pVM);
3601	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3602	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3603
3604	/* WBINVD causes a VM-exit. */
3605	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3606	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3607
3608	/* Enable EPT (aka nested-paging). */
3609	if (pVM->hm.s.fNestedPaging)
3610	fVal \|= VMX_PROC_CTLS2_EPT;
3611
3612	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3613	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3614	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3615	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3616	fVal \|= VMX_PROC_CTLS2_INVPCID;
3617
3618	/* Enable VPID. */
3619	if (pVM->hm.s.vmx.fVpid)
3620	fVal \|= VMX_PROC_CTLS2_VPID;
3621
3622	/* Enable unrestricted guest execution. */
3623	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3624	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3625
3626	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3627	#if 0
3628	/* Enable VMCS shadowing if supported by the hardware and VMX is exposed to the guest. */
3629	if ( pVM->cpum.ro.GuestFeatures.fVmx
3630	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING))
3631	fVal \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
3632	#endif
3633	#endif
3634
3635	#if 0
3636	if (pVM->hm.s.fVirtApicRegs)
3637	{
3638	/* Enable APIC-register virtualization. */
3639	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3640	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3641
3642	/* Enable virtual-interrupt delivery. */
3643	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3644	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3645	}
3646	#endif
3647
3648	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3649	where the TPR shadow resides. */
3650	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3651	* done dynamically. */
3652	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3653	{
3654	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3655	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3656	AssertRCReturn(rc, rc);
3657	}
3658
3659	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3660	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3661	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3662	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3663	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3664
3665	/* Enable Pause-Loop exiting. */
3666	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3667	&& pVM->hm.s.vmx.cPleGapTicks
3668	&& pVM->hm.s.vmx.cPleWindowTicks)
3669	{
3670	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3671
3672	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3673	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3674	AssertRCReturn(rc, rc);
3675	}
3676
3677	if ((fVal & fZap) != fVal)
3678	{
3679	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3680	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3681	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3682	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3683	}
3684
3685	/* Commit it to the VMCS and update our cache. */
3686	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3687	AssertRCReturn(rc, rc);
3688	pVmcsInfo->u32ProcCtls2 = fVal;
3689
3690	return VINF_SUCCESS;
3691	}
3692
3693
3694	/**
3695	* Sets up processor-based VM-execution controls in the VMCS.
3696	*
3697	* @returns VBox status code.
3698	* @param pVCpu The cross context virtual CPU structure.
3699	* @param pVmcsInfo The VMCS info. object.
3700	*/
3701	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3702	{
3703	PVM pVM = pVCpu->CTX_SUFF(pVM);
3704
3705	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3706	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3707
3708	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3709	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3710	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3711	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3712	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3713	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3714	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3715
3716	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3717	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3718	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3719	{
3720	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3721	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3722	}
3723
3724	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3725	if (!pVM->hm.s.fNestedPaging)
3726	{
3727	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3728	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3729	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3730	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3731	}
3732
3733	/* Use TPR shadowing if supported by the CPU. */
3734	if ( PDMHasApic(pVM)
3735	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3736	{
3737	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3738	/* CR8 writes cause a VM-exit based on TPR threshold. */
3739	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3740	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3741	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3742	AssertRCReturn(rc, rc);
3743	}
3744	else
3745	{
3746	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3747	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3748	if (pVM->hm.s.fAllow64BitGuests)
3749	{
3750	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3751	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3752	}
3753	}
3754
3755	/* Use MSR-bitmaps if supported by the CPU. */
3756	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3757	{
3758	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3759	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3760	AssertRCReturn(rc, rc);
3761	}
3762
3763	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3764	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3765	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3766
3767	if ((fVal & fZap) != fVal)
3768	{
3769	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3770	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3771	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3772	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3773	}
3774
3775	/* Commit it to the VMCS and update our cache. */
3776	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3777	AssertRCReturn(rc, rc);
3778	pVmcsInfo->u32ProcCtls = fVal;
3779
3780	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3781	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3782	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3783
3784	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3785	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3786	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3787
3788	/* Sanity check, should not really happen. */
3789	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3790	{ /* likely */ }
3791	else
3792	{
3793	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3794	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3795	}
3796
3797	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3798	return VINF_SUCCESS;
3799	}
3800
3801
3802	/**
3803	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3804	* Processor-based VM-execution) control fields in the VMCS.
3805	*
3806	* @returns VBox status code.
3807	* @param pVCpu The cross context virtual CPU structure.
3808	* @param pVmcsInfo The VMCS info. object.
3809	*
3810	* @remarks Must be called after secondary processor-based VM-execution controls
3811	* have been initialized!
3812	*/
3813	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3814	{
3815	/* Set the VMCS link pointer in the VMCS. */
3816	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3817	if (RT_SUCCESS(rc))
3818	{
3819	/* Set the auto-load/store MSR area addresses in the VMCS. */
3820	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3821	if (RT_SUCCESS(rc))
3822	{
3823	/* Set the CR0/CR4 guest/host mask. */
3824	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3825	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3826	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3827	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3828	if (RT_SUCCESS(rc))
3829	{
3830	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3831	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3832	return VINF_SUCCESS;
3833	}
3834	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3835	}
3836	else
3837	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3838	}
3839	else
3840	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3841	return rc;
3842	}
3843
3844
3845	/**
3846	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3847	*
3848	* We shall setup those exception intercepts that don't change during the
3849	* lifetime of the VM here. The rest are done dynamically while loading the
3850	* guest state.
3851	*
3852	* @returns VBox status code.
3853	* @param pVCpu The cross context virtual CPU structure.
3854	* @param pVmcsInfo The VMCS info. object.
3855	*/
3856	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3857	{
3858	/*
3859	* The following exceptions are always intercepted:
3860	*
3861	* #AC - To prevent the guest from hanging the CPU.
3862	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3863	* recursive #DBs can cause a CPU hang.
3864	* #PF - To sync our shadow page tables when nested-paging is not used.
3865	*/
3866	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3867	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3868	\| RT_BIT(X86_XCPT_DB)
3869	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3870
3871	/* Commit it to the VMCS. */
3872	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3873	AssertRCReturn(rc, rc);
3874
3875	/* Update our cache of the exception bitmap. */
3876	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3877	return VINF_SUCCESS;
3878	}
3879
3880
3881	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3882	/**
3883	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3884	*
3885	* @returns VBox status code.
3886	* @param pVCpu The cross context virtual CPU structure.
3887	* @param pVmcsInfo The VMCS info. object.
3888	*/
3889	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3890	{
3891	PVM pVM = pVCpu->CTX_SUFF(pVM);
3892	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3893	if (RT_SUCCESS(rc))
3894	{
3895	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3896	if (RT_SUCCESS(rc))
3897	{
3898	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3899	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3900	if (RT_SUCCESS(rc))
3901	{
3902	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3903	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3904	if (RT_SUCCESS(rc))
3905	return VINF_SUCCESS;
3906
3907	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3908	}
3909	else
3910	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3911	}
3912	else
3913	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3914	}
3915	else
3916	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3917
3918	return rc;
3919	}
3920	#endif
3921
3922
3923	/**
3924	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3925	* VMX.
3926	*
3927	* @returns VBox status code.
3928	* @param pVCpu The cross context virtual CPU structure.
3929	* @param pVmcsInfo The VMCS info. object.
3930	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3931	*/
3932	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3933	{
3934	Assert(pVmcsInfo->pvVmcs);
3935	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3936
3937	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3938	PVM pVM = pVCpu->CTX_SUFF(pVM);
3939	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3940	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3941
3942	LogFlowFunc(("\n"));
3943
3944	/*
3945	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3946	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3947	*/
3948	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3949	if (RT_SUCCESS(rc))
3950	{
3951	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3952	if (RT_SUCCESS(rc))
3953	{
3954	if (!fIsNstGstVmcs)
3955	{
3956	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3957	if (RT_SUCCESS(rc))
3958	{
3959	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3960	if (RT_SUCCESS(rc))
3961	{
3962	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3963	if (RT_SUCCESS(rc))
3964	{
3965	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3966	if (RT_SUCCESS(rc))
3967	{
3968	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3969	/* If VMCS shadowing is used, initialize the shadow VMCS. */
3970	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
3971	{
3972	Assert(pVmcsInfo->pvShadowVmcs);
3973	VMXVMCSREVID VmcsRevId;
3974	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3975	VmcsRevId.n.fIsShadowVmcs = 1;
3976	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
3977	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
3978	if (RT_SUCCESS(rc))
3979	{ /* likely */ }
3980	else
3981	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
3982	}
3983	#endif
3984	}
3985	else
3986	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3987	}
3988	else
3989	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3990	}
3991	else
3992	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3993	}
3994	else
3995	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3996	}
3997	else
3998	{
3999	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4000	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4001	if (RT_SUCCESS(rc))
4002	{ /* likely */ }
4003	else
4004	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4005	#else
4006	AssertFailed();
4007	#endif
4008	}
4009	}
4010	else
4011	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4012	}
4013	else
4014	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4015
4016	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4017	if (RT_SUCCESS(rc))
4018	{
4019	rc = hmR0VmxClearVmcs(pVmcsInfo);
4020	if (RT_SUCCESS(rc))
4021	{ /* likely */ }
4022	else
4023	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4024	}
4025
4026	/*
4027	* Update the last-error record both for failures and success, so we
4028	* can propagate the status code back to ring-3 for diagnostics.
4029	*/
4030	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4031	NOREF(pszVmcs);
4032	return rc;
4033	}
4034
4035
4036	/**
4037	* Does global VT-x initialization (called during module initialization).
4038	*
4039	* @returns VBox status code.
4040	*/
4041	VMMR0DECL(int) VMXR0GlobalInit(void)
4042	{
4043	#ifdef HMVMX_USE_FUNCTION_TABLE
4044	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4045	# ifdef VBOX_STRICT
4046	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4047	Assert(g_apfnVMExitHandlers[i]);
4048	# endif
4049	#endif
4050	return VINF_SUCCESS;
4051	}
4052
4053
4054	/**
4055	* Does global VT-x termination (called during module termination).
4056	*/
4057	VMMR0DECL(void) VMXR0GlobalTerm()
4058	{
4059	/* Nothing to do currently. */
4060	}
4061
4062
4063	/**
4064	* Sets up and activates VT-x on the current CPU.
4065	*
4066	* @returns VBox status code.
4067	* @param pHostCpu The HM physical-CPU structure.
4068	* @param pVM The cross context VM structure. Can be
4069	* NULL after a host resume operation.
4070	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4071	* fEnabledByHost is @c true).
4072	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4073	* @a fEnabledByHost is @c true).
4074	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4075	* enable VT-x on the host.
4076	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4077	*/
4078	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4079	PCSUPHWVIRTMSRS pHwvirtMsrs)
4080	{
4081	AssertPtr(pHostCpu);
4082	AssertPtr(pHwvirtMsrs);
4083	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4084
4085	/* Enable VT-x if it's not already enabled by the host. */
4086	if (!fEnabledByHost)
4087	{
4088	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4089	if (RT_FAILURE(rc))
4090	return rc;
4091	}
4092
4093	/*
4094	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4095	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4096	* invalidated when flushing by VPID.
4097	*/
4098	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4099	{
4100	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4101	pHostCpu->fFlushAsidBeforeUse = false;
4102	}
4103	else
4104	pHostCpu->fFlushAsidBeforeUse = true;
4105
4106	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4107	++pHostCpu->cTlbFlushes;
4108
4109	return VINF_SUCCESS;
4110	}
4111
4112
4113	/**
4114	* Deactivates VT-x on the current CPU.
4115	*
4116	* @returns VBox status code.
4117	* @param pvCpuPage Pointer to the VMXON region.
4118	* @param HCPhysCpuPage Physical address of the VMXON region.
4119	*
4120	* @remarks This function should never be called when SUPR0EnableVTx() or
4121	* similar was used to enable VT-x on the host.
4122	*/
4123	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4124	{
4125	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4126
4127	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4128	return hmR0VmxLeaveRootMode();
4129	}
4130
4131
4132	/**
4133	* Does per-VM VT-x initialization.
4134	*
4135	* @returns VBox status code.
4136	* @param pVM The cross context VM structure.
4137	*/
4138	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4139	{
4140	AssertPtr(pVM);
4141	LogFlowFunc(("pVM=%p\n", pVM));
4142
4143	int rc = hmR0VmxStructsAlloc(pVM);
4144	if (RT_FAILURE(rc))
4145	{
4146	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4147	return rc;
4148	}
4149
4150	return VINF_SUCCESS;
4151	}
4152
4153
4154	/**
4155	* Does per-VM VT-x termination.
4156	*
4157	* @returns VBox status code.
4158	* @param pVM The cross context VM structure.
4159	*/
4160	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4161	{
4162	AssertPtr(pVM);
4163	LogFlowFunc(("pVM=%p\n", pVM));
4164
4165	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4166	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4167	{
4168	Assert(pVM->hm.s.vmx.pvScratch);
4169	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4170	}
4171	#endif
4172	hmR0VmxStructsFree(pVM);
4173	return VINF_SUCCESS;
4174	}
4175
4176
4177	/**
4178	* Sets up the VM for execution using hardware-assisted VMX.
4179	* This function is only called once per-VM during initialization.
4180	*
4181	* @returns VBox status code.
4182	* @param pVM The cross context VM structure.
4183	*/
4184	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4185	{
4186	AssertPtr(pVM);
4187	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4188
4189	LogFlowFunc(("pVM=%p\n", pVM));
4190
4191	/*
4192	* At least verify if VMX is enabled, since we can't check if we're in
4193	* VMX root mode or not without causing a #GP.
4194	*/
4195	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4196	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4197	{ /* likely */ }
4198	else
4199	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4200
4201	/*
4202	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4203	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4204	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4205	*/
4206	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4207	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4208	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4209	{
4210	LogRelFunc(("Invalid real-on-v86 state.\n"));
4211	return VERR_INTERNAL_ERROR;
4212	}
4213
4214	/* Initialize these always, see hmR3InitFinalizeR0().*/
4215	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4216	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4217
4218	/* Setup the tagged-TLB flush handlers. */
4219	int rc = hmR0VmxSetupTaggedTlb(pVM);
4220	if (RT_FAILURE(rc))
4221	{
4222	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
4223	return rc;
4224	}
4225
4226	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
4227	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
4228	#if HC_ARCH_BITS == 64
4229	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4230	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
4231	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
4232	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
4233	#endif
4234
4235	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4236	{
4237	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4238	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4239
4240	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4241	if (RT_SUCCESS(rc))
4242	{
4243	#if HC_ARCH_BITS == 32
4244	hmR0VmxInitVmcsReadCache(pVCpu);
4245	#endif
4246	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4247	if (pVM->cpum.ro.GuestFeatures.fVmx)
4248	{
4249	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4250	if (RT_SUCCESS(rc))
4251	{ /* likely */ }
4252	else
4253	{
4254	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4255	return rc;
4256	}
4257	}
4258	#endif
4259	}
4260	else
4261	{
4262	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4263	return rc;
4264	}
4265	}
4266
4267	return VINF_SUCCESS;
4268	}
4269
4270
4271	#if HC_ARCH_BITS == 32
4272	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4273	/**
4274	* Check if guest state allows safe use of 32-bit switcher again.
4275	*
4276	* Segment bases and protected mode structures must be 32-bit addressable
4277	* because the 32-bit switcher will ignore high dword when writing these VMCS
4278	* fields. See @bugref{8432} for details.
4279	*
4280	* @returns true if safe, false if must continue to use the 64-bit switcher.
4281	* @param pCtx Pointer to the guest-CPU context.
4282	*
4283	* @remarks No-long-jump zone!!!
4284	*/
4285	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4286	{
4287	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4288	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4289	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4290	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4291	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4292	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4293	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4294	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4295	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4296	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4297
4298	/* All good, bases are 32-bit. */
4299	return true;
4300	}
4301	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4302
4303	# ifdef VBOX_STRICT
4304	static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4305	{
4306	switch (idxField)
4307	{
4308	case VMX_VMCS_GUEST_RIP:
4309	case VMX_VMCS_GUEST_RSP:
4310	case VMX_VMCS_GUEST_SYSENTER_EIP:
4311	case VMX_VMCS_GUEST_SYSENTER_ESP:
4312	case VMX_VMCS_GUEST_GDTR_BASE:
4313	case VMX_VMCS_GUEST_IDTR_BASE:
4314	case VMX_VMCS_GUEST_CS_BASE:
4315	case VMX_VMCS_GUEST_DS_BASE:
4316	case VMX_VMCS_GUEST_ES_BASE:
4317	case VMX_VMCS_GUEST_FS_BASE:
4318	case VMX_VMCS_GUEST_GS_BASE:
4319	case VMX_VMCS_GUEST_SS_BASE:
4320	case VMX_VMCS_GUEST_LDTR_BASE:
4321	case VMX_VMCS_GUEST_TR_BASE:
4322	case VMX_VMCS_GUEST_CR3:
4323	return true;
4324	}
4325	return false;
4326	}
4327
4328	static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4329	{
4330	switch (idxField)
4331	{
4332	/* Read-only fields. */
4333	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4334	return true;
4335	}
4336	/* Remaining readable fields should also be writable. */
4337	return hmR0VmxIsValidWriteFieldInCache(idxField);
4338	}
4339	# endif /* VBOX_STRICT */
4340
4341
4342	/**
4343	* Executes the specified handler in 64-bit mode.
4344	*
4345	* @returns VBox status code (no informational status codes).
4346	* @param pVCpu The cross context virtual CPU structure.
4347	* @param enmOp The operation to perform.
4348	* @param cParams Number of parameters.
4349	* @param paParam Array of 32-bit parameters.
4350	*/
4351	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4352	{
4353	AssertPtr(pVCpu);
4354	PVM pVM = pVCpu->CTX_SUFF(pVM);
4355	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4356	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4357	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4358	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4359
4360	#ifdef VBOX_STRICT
4361	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4362	Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4363
4364	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4365	Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4366	#endif
4367
4368	/* Disable interrupts. */
4369	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4370
4371	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4372	RTCPUID idHostCpu = RTMpCpuId();
4373	CPUMR0SetLApic(pVCpu, idHostCpu);
4374	#endif
4375
4376	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4377
4378	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4379	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4380
4381	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4382	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4383	hmR0VmxClearVmcs(pVmcsInfo);
4384
4385	/* Leave VMX root mode and disable VMX. */
4386	VMXDisable();
4387	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4388
4389	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4390	CPUMSetHyperEIP(pVCpu, enmOp);
4391	for (int i = (int)cParams - 1; i >= 0; i--)
4392	CPUMPushHyper(pVCpu, paParam[i]);
4393
4394	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4395
4396	/* Call the switcher. */
4397	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4398	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4399
4400	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4401	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4402
4403	/* Re-enter VMX root mode. */
4404	int rc2 = VMXEnable(HCPhysCpuPage);
4405	if (RT_FAILURE(rc2))
4406	{
4407	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4408	ASMSetFlags(fOldEFlags);
4409	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4410	return rc2;
4411	}
4412
4413	/* Restore the VMCS as the current VMCS. */
4414	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4415	AssertRC(rc2);
4416	Assert(!(ASMGetFlags() & X86_EFL_IF));
4417	ASMSetFlags(fOldEFlags);
4418	return rc;
4419	}
4420
4421
4422	/**
4423	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4424	* supporting 64-bit guests.
4425	*
4426	* @returns VBox status code.
4427	* @param fResume Whether to VMLAUNCH or VMRESUME.
4428	* @param pCtx Pointer to the guest-CPU context.
4429	* @param pCache Pointer to the VMCS batch cache.
4430	* @param pVM The cross context VM structure.
4431	* @param pVCpu The cross context virtual CPU structure.
4432	*/
4433	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4434	{
4435	NOREF(fResume);
4436
4437	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4438	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4439	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4440
4441	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4442	pCache->uPos = 1;
4443	pCache->interPD = PGMGetInterPaeCR3(pVM);
4444	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4445	#endif
4446
4447	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4448	pCache->TestIn.HCPhysCpuPage = 0;
4449	pCache->TestIn.HCPhysVmcs = 0;
4450	pCache->TestIn.pCache = 0;
4451	pCache->TestOut.HCPhysVmcs = 0;
4452	pCache->TestOut.pCache = 0;
4453	pCache->TestOut.pCtx = 0;
4454	pCache->TestOut.eflags = 0;
4455	#else
4456	NOREF(pCache);
4457	#endif
4458
4459	uint32_t aParam[10];
4460	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4461	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4462	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4463	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4464	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4465	aParam[5] = 0;
4466	aParam[6] = VM_RC_ADDR(pVM, pVM);
4467	aParam[7] = 0;
4468	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4469	aParam[9] = 0;
4470
4471	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4472	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4473	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4474	#endif
4475	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4476
4477	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4478	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4479	Assert(pCtx->dr[4] == 10);
4480	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4481	#endif
4482
4483	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4484	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4485	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4486	pVmcsInfo->HCPhysVmcs));
4487	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4488	pCache->TestOut.HCPhysVmcs));
4489	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4490	pCache->TestOut.pCache));
4491	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4492	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4493	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4494	pCache->TestOut.pCtx));
4495	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4496	#endif
4497	NOREF(pCtx);
4498	return rc;
4499	}
4500	#endif
4501
4502
4503	/**
4504	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4505	* the VMCS.
4506	*
4507	* @returns VBox status code.
4508	*/
4509	static int hmR0VmxExportHostControlRegs(void)
4510	{
4511	RTCCUINTREG uReg = ASMGetCR0();
4512	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4513	AssertRCReturn(rc, rc);
4514
4515	uReg = ASMGetCR3();
4516	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4517	AssertRCReturn(rc, rc);
4518
4519	uReg = ASMGetCR4();
4520	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4521	AssertRCReturn(rc, rc);
4522	return rc;
4523	}
4524
4525
4526	/**
4527	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4528	* the host-state area in the VMCS.
4529	*
4530	* @returns VBox status code.
4531	* @param pVCpu The cross context virtual CPU structure.
4532	*/
4533	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4534	{
4535	#if HC_ARCH_BITS == 64
4536	/**
4537	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4538	* requirements. See hmR0VmxExportHostSegmentRegs().
4539	*/
4540	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4541	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4542	{ \
4543	bool fValidSelector = true; \
4544	if ((selValue) & X86_SEL_LDT) \
4545	{ \
4546	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4547	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4548	} \
4549	if (fValidSelector) \
4550	{ \
4551	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4552	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4553	} \
4554	(selValue) = 0; \
4555	}
4556
4557	/*
4558	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4559	* will be messed up. We should -not- save the messed up state without restoring
4560	* the original host-state, see @bugref{7240}.
4561	*
4562	* This apparently can happen (most likely the FPU changes), deal with it rather than
4563	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4564	*/
4565	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4566	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4567	{
4568	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4569	pVCpu->idCpu));
4570	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4571	}
4572	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4573	#else
4574	RT_NOREF(pVCpu);
4575	#endif
4576
4577	/*
4578	* Host DS, ES, FS and GS segment registers.
4579	*/
4580	#if HC_ARCH_BITS == 64
4581	RTSEL uSelDS = ASMGetDS();
4582	RTSEL uSelES = ASMGetES();
4583	RTSEL uSelFS = ASMGetFS();
4584	RTSEL uSelGS = ASMGetGS();
4585	#else
4586	RTSEL uSelDS = 0;
4587	RTSEL uSelES = 0;
4588	RTSEL uSelFS = 0;
4589	RTSEL uSelGS = 0;
4590	#endif
4591
4592	/*
4593	* Host CS and SS segment registers.
4594	*/
4595	RTSEL uSelCS = ASMGetCS();
4596	RTSEL uSelSS = ASMGetSS();
4597
4598	/*
4599	* Host TR segment register.
4600	*/
4601	RTSEL uSelTR = ASMGetTR();
4602
4603	#if HC_ARCH_BITS == 64
4604	/*
4605	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4606	* gain VM-entry and restore them before we get preempted.
4607	*
4608	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4609	*/
4610	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4611	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4612	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4613	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4614	# undef VMXLOCAL_ADJUST_HOST_SEG
4615	#endif
4616
4617	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4618	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4619	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4620	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4621	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4622	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4623	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4624	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4625	Assert(uSelCS);
4626	Assert(uSelTR);
4627
4628	/* Write these host selector fields into the host-state area in the VMCS. */
4629	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4630	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4631	#if HC_ARCH_BITS == 64
4632	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4633	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4634	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4635	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4636	#else
4637	NOREF(uSelDS);
4638	NOREF(uSelES);
4639	NOREF(uSelFS);
4640	NOREF(uSelGS);
4641	#endif
4642	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4643	AssertRCReturn(rc, rc);
4644
4645	/*
4646	* Host GDTR and IDTR.
4647	*/
4648	RTGDTR Gdtr;
4649	RTIDTR Idtr;
4650	RT_ZERO(Gdtr);
4651	RT_ZERO(Idtr);
4652	ASMGetGDTR(&Gdtr);
4653	ASMGetIDTR(&Idtr);
4654	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4655	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4656	AssertRCReturn(rc, rc);
4657
4658	#if HC_ARCH_BITS == 64
4659	/*
4660	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4661	* them to the maximum limit (0xffff) on every VM-exit.
4662	*/
4663	if (Gdtr.cbGdt != 0xffff)
4664	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4665
4666	/*
4667	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4668	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4669	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4670	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4671	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4672	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4673	* at 0xffff on hosts where we are sure it won't cause trouble.
4674	*/
4675	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4676	if (Idtr.cbIdt < 0x0fff)
4677	# else
4678	if (Idtr.cbIdt != 0xffff)
4679	# endif
4680	{
4681	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4682	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4683	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4684	}
4685	#endif
4686
4687	/*
4688	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4689	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4690	* RPL should be too in most cases.
4691	*/
4692	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4693	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4694
4695	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4696	#if HC_ARCH_BITS == 64
4697	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4698
4699	/*
4700	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4701	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4702	* restoration if the host has something else. Task switching is not supported in 64-bit
4703	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4704	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4705	*
4706	* [1] See Intel spec. 3.5 "System Descriptor Types".
4707	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4708	*/
4709	PVM pVM = pVCpu->CTX_SUFF(pVM);
4710	Assert(pDesc->System.u4Type == 11);
4711	if ( pDesc->System.u16LimitLow != 0x67
4712	\|\| pDesc->System.u4LimitHigh)
4713	{
4714	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4715	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4716	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4717	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4718	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4719	}
4720
4721	/*
4722	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4723	*/
4724	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4725	{
4726	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4727	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4728	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4729	{
4730	/* The GDT is read-only but the writable GDT is available. */
4731	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4732	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4733	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4734	AssertRCReturn(rc, rc);
4735	}
4736	}
4737	#else
4738	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4739	#endif
4740	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4741	AssertRCReturn(rc, rc);
4742
4743	/*
4744	* Host FS base and GS base.
4745	*/
4746	#if HC_ARCH_BITS == 64
4747	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4748	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4749	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4750	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4751	AssertRCReturn(rc, rc);
4752
4753	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4754	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4755	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4756	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4757	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4758	#endif
4759	return VINF_SUCCESS;
4760	}
4761
4762
4763	/**
4764	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4765	* host-state area of the VMCS.
4766	*
4767	* These MSRs will be automatically restored on the host after every successful
4768	* VM-exit.
4769	*
4770	* @returns VBox status code.
4771	* @param pVCpu The cross context virtual CPU structure.
4772	*
4773	* @remarks No-long-jump zone!!!
4774	*/
4775	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4776	{
4777	AssertPtr(pVCpu);
4778
4779	/*
4780	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4781	* rather than swapping them on every VM-entry.
4782	*/
4783	hmR0VmxLazySaveHostMsrs(pVCpu);
4784
4785	/*
4786	* Host Sysenter MSRs.
4787	*/
4788	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4789	#if HC_ARCH_BITS == 32
4790	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4791	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4792	#else
4793	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4794	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4795	#endif
4796	AssertRCReturn(rc, rc);
4797
4798	/*
4799	* Host EFER MSR.
4800	*
4801	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4802	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4803	*/
4804	PVM pVM = pVCpu->CTX_SUFF(pVM);
4805	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4806	{
4807	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4808	AssertRCReturn(rc, rc);
4809	}
4810
4811	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4812	* hmR0VmxExportGuestEntryExitCtls(). */
4813
4814	return VINF_SUCCESS;
4815	}
4816
4817
4818	/**
4819	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4820	*
4821	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4822	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4823	*
4824	* @returns true if we need to load guest EFER, false otherwise.
4825	* @param pVCpu The cross context virtual CPU structure.
4826	*
4827	* @remarks Requires EFER, CR4.
4828	* @remarks No-long-jump zone!!!
4829	*/
4830	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4831	{
4832	#ifdef HMVMX_ALWAYS_SWAP_EFER
4833	RT_NOREF(pVCpu);
4834	return true;
4835	#else
4836	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4837	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4838	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4839	if (CPUMIsGuestInLongModeEx(pCtx))
4840	return false;
4841	#endif
4842
4843	PVM pVM = pVCpu->CTX_SUFF(pVM);
4844	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4845	uint64_t const u64GuestEfer = pCtx->msrEFER;
4846
4847	/*
4848	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4849	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4850	*/
4851	if ( CPUMIsGuestInLongModeEx(pCtx)
4852	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4853	return true;
4854
4855	/*
4856	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4857	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4858	*
4859	* See Intel spec. 4.5 "IA-32e Paging".
4860	* See Intel spec. 4.1.1 "Three Paging Modes".
4861	*
4862	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4863	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4864	*/
4865	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4866	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4867	if ( (pCtx->cr4 & X86_CR4_PAE)
4868	&& (pCtx->cr0 & X86_CR0_PG)
4869	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4870	{
4871	/* Assert that host is NX capable. */
4872	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4873	return true;
4874	}
4875
4876	return false;
4877	#endif
4878	}
4879
4880	/**
4881	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4882	* VMCS.
4883	*
4884	* This is typically required when the guest changes paging mode.
4885	*
4886	* @returns VBox status code.
4887	* @param pVCpu The cross context virtual CPU structure.
4888	* @param pVmxTransient The VMX-transient structure.
4889	*
4890	* @remarks Requires EFER.
4891	* @remarks No-long-jump zone!!!
4892	*/
4893	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4894	{
4895	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4896	{
4897	PVM pVM = pVCpu->CTX_SUFF(pVM);
4898	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4899
4900	/*
4901	* VM-entry controls.
4902	*/
4903	{
4904	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4905	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4906
4907	/*
4908	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4909	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4910	*
4911	* For nested-guests, this is a mandatory VM-entry control. It's also
4912	* required because we do not want to leak host bits to the nested-guest.
4913	*/
4914	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4915
4916	/*
4917	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4918	*
4919	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4920	* required to get the nested-guest working with hardware-assisted VMX execution.
4921	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4922	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4923	* here rather than while merging the guest VMCS controls.
4924	*/
4925	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4926	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4927	else
4928	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4929
4930	/*
4931	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4932	*
4933	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4934	* regardless of whether the nested-guest VMCS specifies it because we are free to
4935	* load whatever MSRs we require and we do not need to modify the guest visible copy
4936	* of the VM-entry MSR load area.
4937	*/
4938	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4939	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4940	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4941	else
4942	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4943
4944	/*
4945	* The following should -not- be set (since we're not in SMM mode):
4946	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4947	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4948	*/
4949
4950	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4951	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4952
4953	if ((fVal & fZap) == fVal)
4954	{ /* likely */ }
4955	else
4956	{
4957	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4958	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4959	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4960	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4961	}
4962
4963	/* Commit it to the VMCS. */
4964	if (pVmcsInfo->u32EntryCtls != fVal)
4965	{
4966	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4967	AssertRCReturn(rc, rc);
4968	pVmcsInfo->u32EntryCtls = fVal;
4969	}
4970	}
4971
4972	/*
4973	* VM-exit controls.
4974	*/
4975	{
4976	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4977	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4978
4979	/*
4980	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4981	* supported the 1-setting of this bit.
4982	*
4983	* For nested-guests, we set the "save debug controls" as the converse
4984	* "load debug controls" is mandatory for nested-guests anyway.
4985	*/
4986	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4987
4988	/*
4989	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4990	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4991	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4992	* hmR0VmxExportHostMsrs().
4993	*
4994	* For nested-guests, we always set this bit as we do not support 32-bit
4995	* hosts.
4996	*/
4997	#if HC_ARCH_BITS == 64
4998	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4999	#else
5000	Assert(!pVmxTransient->fIsNestedGuest);
5001	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5002	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5003	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5004	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5005	{
5006	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5007	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5008	}
5009	else
5010	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5011	#endif
5012
5013	/*
5014	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
5015	*
5016	* For nested-guests, we should use the "save IA32_EFER" control if we also
5017	* used the "load IA32_EFER" control while exporting VM-entry controls.
5018	*/
5019	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5020	&& hmR0VmxShouldSwapEferMsr(pVCpu))
5021	{
5022	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
5023	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
5024	}
5025
5026	/*
5027	* Enable saving of the VMX-preemption timer value on VM-exit.
5028	* For nested-guests, currently not exposed/used.
5029	*/
5030	if ( pVM->hm.s.vmx.fUsePreemptTimer
5031	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5032	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5033
5034	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5035	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5036
5037	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5038	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
5039	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5040
5041	if ((fVal & fZap) == fVal)
5042	{ /* likely */ }
5043	else
5044	{
5045	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5046	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5047	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5048	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5049	}
5050
5051	/* Commit it to the VMCS. */
5052	if (pVmcsInfo->u32ExitCtls != fVal)
5053	{
5054	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5055	AssertRCReturn(rc, rc);
5056	pVmcsInfo->u32ExitCtls = fVal;
5057	}
5058	}
5059
5060	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5061	}
5062	return VINF_SUCCESS;
5063	}
5064
5065
5066	/**
5067	* Sets the TPR threshold in the VMCS.
5068	*
5069	* @returns VBox status code.
5070	* @param pVCpu The cross context virtual CPU structure.
5071	* @param pVmcsInfo The VMCS info. object.
5072	* @param u32TprThreshold The TPR threshold (task-priority class only).
5073	*/
5074	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5075	{
5076	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5077	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5078	RT_NOREF2(pVCpu, pVmcsInfo);
5079	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5080	}
5081
5082
5083	/**
5084	* Exports the guest APIC TPR state into the VMCS.
5085	*
5086	* @returns VBox status code.
5087	* @param pVCpu The cross context virtual CPU structure.
5088	* @param pVmxTransient The VMX-transient structure.
5089	*
5090	* @remarks No-long-jump zone!!!
5091	*/
5092	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5093	{
5094	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5095	{
5096	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5097
5098	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5099	if (!pVmxTransient->fIsNestedGuest)
5100	{
5101	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5102	&& APICIsEnabled(pVCpu))
5103	{
5104	/*
5105	* Setup TPR shadowing.
5106	*/
5107	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5108	{
5109	bool fPendingIntr = false;
5110	uint8_t u8Tpr = 0;
5111	uint8_t u8PendingIntr = 0;
5112	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5113	AssertRCReturn(rc, rc);
5114
5115	/*
5116	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5117	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5118	* priority of the pending interrupt so we can deliver the interrupt. If there
5119	* are no interrupts pending, set threshold to 0 to not cause any
5120	* TPR-below-threshold VM-exits.
5121	*/
5122	Assert(pVmcsInfo->pbVirtApic);
5123	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5124	uint32_t u32TprThreshold = 0;
5125	if (fPendingIntr)
5126	{
5127	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5128	(which is the Task-Priority Class). */
5129	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5130	const uint8_t u8TprPriority = u8Tpr >> 4;
5131	if (u8PendingPriority <= u8TprPriority)
5132	u32TprThreshold = u8PendingPriority;
5133	}
5134
5135	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5136	AssertRCReturn(rc, rc);
5137	}
5138	}
5139	}
5140	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5141	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5142	}
5143	return VINF_SUCCESS;
5144	}
5145
5146
5147	/**
5148	* Gets the guest interruptibility-state.
5149	*
5150	* @returns Guest's interruptibility-state.
5151	* @param pVCpu The cross context virtual CPU structure.
5152	* @param pVmxTransient The VMX-transient structure.
5153	*
5154	* @remarks No-long-jump zone!!!
5155	*/
5156	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5157	{
5158	/*
5159	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5160	*/
5161	uint32_t fIntrState = 0;
5162	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5163	{
5164	/* If inhibition is active, RIP and RFLAGS should've been updated
5165	(i.e. read previously from the VMCS or from ring-3). */
5166	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5167	#ifdef VBOX_STRICT
5168	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5169	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5170	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5171	#endif
5172	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5173	{
5174	if (pCtx->eflags.Bits.u1IF)
5175	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5176	else
5177	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5178	}
5179	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5180	{
5181	/*
5182	* We can clear the inhibit force flag as even if we go back to the recompiler
5183	* without executing guest code in VT-x, the flag's condition to be cleared is
5184	* met and thus the cleared state is correct.
5185	*/
5186	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5187	}
5188	}
5189
5190	/*
5191	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5192	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5193	* setting this would block host-NMIs and IRET will not clear the blocking.
5194	*
5195	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5196	*
5197	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5198	*/
5199	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5200	&& CPUMIsGuestNmiBlocking(pVCpu))
5201	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5202
5203	return fIntrState;
5204	}
5205
5206
5207	/**
5208	* Exports the exception intercepts required for guest execution in the VMCS.
5209	*
5210	* @returns VBox status code.
5211	* @param pVCpu The cross context virtual CPU structure.
5212	* @param pVmxTransient The VMX-transient structure.
5213	*
5214	* @remarks No-long-jump zone!!!
5215	*/
5216	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5217	{
5218	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5219	{
5220	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5221	if ( !pVmxTransient->fIsNestedGuest
5222	&& pVCpu->hm.s.fGIMTrapXcptUD)
5223	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5224	else
5225	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5226
5227	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5228	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5229	}
5230	return VINF_SUCCESS;
5231	}
5232
5233
5234	/**
5235	* Exports the guest's RIP into the guest-state area in the VMCS.
5236	*
5237	* @returns VBox status code.
5238	* @param pVCpu The cross context virtual CPU structure.
5239	*
5240	* @remarks No-long-jump zone!!!
5241	*/
5242	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5243	{
5244	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5245	{
5246	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5247
5248	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5249	AssertRCReturn(rc, rc);
5250
5251	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5252	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5253	}
5254	return VINF_SUCCESS;
5255	}
5256
5257
5258	/**
5259	* Exports the guest's RSP into the guest-state area in the VMCS.
5260	*
5261	* @returns VBox status code.
5262	* @param pVCpu The cross context virtual CPU structure.
5263	*
5264	* @remarks No-long-jump zone!!!
5265	*/
5266	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5267	{
5268	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5269	{
5270	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5271
5272	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5273	AssertRCReturn(rc, rc);
5274
5275	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5276	}
5277	return VINF_SUCCESS;
5278	}
5279
5280
5281	/**
5282	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5283	*
5284	* @returns VBox status code.
5285	* @param pVCpu The cross context virtual CPU structure.
5286	* @param pVmxTransient The VMX-transient structure.
5287	*
5288	* @remarks No-long-jump zone!!!
5289	*/
5290	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5291	{
5292	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5293	{
5294	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5295
5296	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5297	Let us assert it as such and use 32-bit VMWRITE. */
5298	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5299	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5300	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5301	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5302
5303	/*
5304	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5305	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5306	* can run the real-mode guest code under Virtual 8086 mode.
5307	*/
5308	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5309	if (pVmcsInfo->RealMode.fRealOnV86Active)
5310	{
5311	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5312	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5313	Assert(!pVmxTransient->fIsNestedGuest);
5314	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5315	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5316	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5317	}
5318
5319	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5320	AssertRCReturn(rc, rc);
5321
5322	/*
5323	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5324	*
5325	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5326	* through the hypervisor debugger using EFLAGS.TF.
5327	*/
5328	if ( !pVmxTransient->fIsNestedGuest
5329	&& !pVCpu->hm.s.fSingleInstruction
5330	&& fEFlags.Bits.u1TF)
5331	{
5332	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5333	* premature trips to ring-3 esp since IEM does not yet handle it. */
5334	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5335	AssertRCReturn(rc, rc);
5336	}
5337	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5338	* nested-guest VMCS. */
5339
5340	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5341	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
5342	}
5343	return VINF_SUCCESS;
5344	}
5345
5346
5347	/**
5348	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5349	*
5350	* The guest FPU state is always pre-loaded hence we don't need to bother about
5351	* sharing FPU related CR0 bits between the guest and host.
5352	*
5353	* @returns VBox status code.
5354	* @param pVCpu The cross context virtual CPU structure.
5355	* @param pVmxTransient The VMX-transient structure.
5356	*
5357	* @remarks No-long-jump zone!!!
5358	*/
5359	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5360	{
5361	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5362	{
5363	PVM pVM = pVCpu->CTX_SUFF(pVM);
5364	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5365
5366	/*
5367	* Figure out fixed CR0 bits in VMX operation.
5368	*/
5369	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5370	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5371	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5372	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5373	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5374	else
5375	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5376
5377	if (!pVmxTransient->fIsNestedGuest)
5378	{
5379	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5380	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5381	uint64_t const u64ShadowCr0 = u64GuestCr0;
5382	Assert(!RT_HI_U32(u64GuestCr0));
5383
5384	/*
5385	* Setup VT-x's view of the guest CR0.
5386	*/
5387	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5388	if (pVM->hm.s.fNestedPaging)
5389	{
5390	if (CPUMIsGuestPagingEnabled(pVCpu))
5391	{
5392	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5393	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5394	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5395	}
5396	else
5397	{
5398	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5399	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5400	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5401	}
5402
5403	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5404	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5405	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5406	}
5407	else
5408	{
5409	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5410	u64GuestCr0 \|= X86_CR0_WP;
5411	}
5412
5413	/*
5414	* Guest FPU bits.
5415	*
5416	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5417	* using CR0.TS.
5418	*
5419	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5420	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5421	*/
5422	u64GuestCr0 \|= X86_CR0_NE;
5423
5424	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5425	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5426
5427	/*
5428	* Update exception intercepts.
5429	*/
5430	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5431	if (pVmcsInfo->RealMode.fRealOnV86Active)
5432	{
5433	Assert(PDMVmmDevHeapIsEnabled(pVM));
5434	Assert(pVM->hm.s.vmx.pRealModeTSS);
5435	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5436	}
5437	else
5438	{
5439	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5440	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5441	if (fInterceptMF)
5442	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5443	}
5444
5445	/* Additional intercepts for debugging, define these yourself explicitly. */
5446	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5447	uXcptBitmap \|= 0
5448	\| RT_BIT(X86_XCPT_BP)
5449	\| RT_BIT(X86_XCPT_DE)
5450	\| RT_BIT(X86_XCPT_NM)
5451	\| RT_BIT(X86_XCPT_TS)
5452	\| RT_BIT(X86_XCPT_UD)
5453	\| RT_BIT(X86_XCPT_NP)
5454	\| RT_BIT(X86_XCPT_SS)
5455	\| RT_BIT(X86_XCPT_GP)
5456	\| RT_BIT(X86_XCPT_PF)
5457	\| RT_BIT(X86_XCPT_MF)
5458	;
5459	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5460	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5461	#endif
5462	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5463	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5464	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5465
5466	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5467	u64GuestCr0 \|= fSetCr0;
5468	u64GuestCr0 &= fZapCr0;
5469	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5470
5471	/* Commit the CR0 and related fields to the guest VMCS. */
5472	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5473	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5474	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5475	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5476	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5477	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5478	AssertRCReturn(rc, rc);
5479
5480	/* Update our caches. */
5481	pVmcsInfo->u32ProcCtls = uProcCtls;
5482	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5483
5484	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5485	}
5486	else
5487	{
5488	/*
5489	* With nested-guests, we may have extended the guest/host mask here since we
5490	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5491	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5492	* originally supplied. We must copy those bits from the nested-guest CR0 into
5493	* the nested-guest CR0 read-shadow.
5494	*/
5495	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5496	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5497	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5498	Assert(!RT_HI_U32(u64GuestCr0));
5499	Assert(u64GuestCr0 & X86_CR0_NE);
5500
5501	/*
5502	* Apply the hardware specified fixed CR0 bits and enable caching.
5503	* Note! We could be altering our VMX emulation's fixed bits. We thus
5504	* need to re-apply them while importing CR0.
5505	*/
5506	u64GuestCr0 \|= fSetCr0;
5507	u64GuestCr0 &= fZapCr0;
5508	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5509
5510	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5511	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5512	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5513	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5514	AssertRCReturn(rc, rc);
5515
5516	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5517	}
5518
5519	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5520	}
5521
5522	return VINF_SUCCESS;
5523	}
5524
5525
5526	/**
5527	* Exports the guest control registers (CR3, CR4) into the guest-state area
5528	* in the VMCS.
5529	*
5530	* @returns VBox strict status code.
5531	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5532	* without unrestricted guest access and the VMMDev is not presently
5533	* mapped (e.g. EFI32).
5534	*
5535	* @param pVCpu The cross context virtual CPU structure.
5536	* @param pVmxTransient The VMX-transient structure.
5537	*
5538	* @remarks No-long-jump zone!!!
5539	*/
5540	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5541	{
5542	int rc = VINF_SUCCESS;
5543	PVM pVM = pVCpu->CTX_SUFF(pVM);
5544
5545	/*
5546	* Guest CR2.
5547	* It's always loaded in the assembler code. Nothing to do here.
5548	*/
5549
5550	/*
5551	* Guest CR3.
5552	*/
5553	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5554	{
5555	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5556
5557	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5558	if (pVM->hm.s.fNestedPaging)
5559	{
5560	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5561	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5562
5563	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5564	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5565	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5566	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5567
5568	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5569	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5570	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5571
5572	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5573	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5574	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5575	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5576	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5577	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5578	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5579
5580	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5581	AssertRCReturn(rc, rc);
5582
5583	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5584	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5585	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5586	{
5587	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5588	if (CPUMIsGuestInPAEModeEx(pCtx))
5589	{
5590	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5591	AssertRCReturn(rc, rc);
5592	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5593	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5594	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5595	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5596	AssertRCReturn(rc, rc);
5597	}
5598
5599	/*
5600	* The guest's view of its CR3 is unblemished with nested paging when the
5601	* guest is using paging or we have unrestricted guest execution to handle
5602	* the guest when it's not using paging.
5603	*/
5604	GCPhysGuestCr3 = pCtx->cr3;
5605	}
5606	else
5607	{
5608	/*
5609	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5610	* thinks it accesses physical memory directly, we use our identity-mapped
5611	* page table to map guest-linear to guest-physical addresses. EPT takes care
5612	* of translating it to host-physical addresses.
5613	*/
5614	RTGCPHYS GCPhys;
5615	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5616
5617	/* We obtain it here every time as the guest could have relocated this PCI region. */
5618	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5619	if (RT_SUCCESS(rc))
5620	{ /* likely */ }
5621	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5622	{
5623	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5624	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5625	}
5626	else
5627	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5628
5629	GCPhysGuestCr3 = GCPhys;
5630	}
5631
5632	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5633	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5634	AssertRCReturn(rc, rc);
5635	}
5636	else
5637	{
5638	/* Non-nested paging case, just use the hypervisor's CR3. */
5639	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5640
5641	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5642	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5643	AssertRCReturn(rc, rc);
5644	}
5645
5646	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5647	}
5648
5649	/*
5650	* Guest CR4.
5651	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5652	*/
5653	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5654	{
5655	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5656	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5657
5658	/*
5659	* Figure out fixed CR4 bits in VMX operation.
5660	*/
5661	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5662	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5663	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5664
5665	/*
5666	* With nested-guests, we may have extended the guest/host mask here (since we
5667	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5668	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5669	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5670	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5671	*/
5672	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5673	uint64_t u64GuestCr4 = pCtx->cr4;
5674	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5675	? pCtx->cr4
5676	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5677	Assert(!RT_HI_U32(u64GuestCr4));
5678
5679	/*
5680	* Setup VT-x's view of the guest CR4.
5681	*
5682	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5683	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5684	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5685	*
5686	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5687	*/
5688	if (pVmcsInfo->RealMode.fRealOnV86Active)
5689	{
5690	Assert(pVM->hm.s.vmx.pRealModeTSS);
5691	Assert(PDMVmmDevHeapIsEnabled(pVM));
5692	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5693	}
5694
5695	if (pVM->hm.s.fNestedPaging)
5696	{
5697	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5698	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5699	{
5700	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5701	u64GuestCr4 \|= X86_CR4_PSE;
5702	/* Our identity mapping is a 32-bit page directory. */
5703	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5704	}
5705	/* else use guest CR4.*/
5706	}
5707	else
5708	{
5709	Assert(!pVmxTransient->fIsNestedGuest);
5710
5711	/*
5712	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5713	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5714	*/
5715	switch (pVCpu->hm.s.enmShadowMode)
5716	{
5717	case PGMMODE_REAL: /* Real-mode. */
5718	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5719	case PGMMODE_32_BIT: /* 32-bit paging. */
5720	{
5721	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5722	break;
5723	}
5724
5725	case PGMMODE_PAE: /* PAE paging. */
5726	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5727	{
5728	u64GuestCr4 \|= X86_CR4_PAE;
5729	break;
5730	}
5731
5732	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5733	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5734	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5735	break;
5736	#endif
5737	default:
5738	AssertFailed();
5739	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5740	}
5741	}
5742
5743	/*
5744	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
5745	* Note! For nested-guests, we could be altering our VMX emulation's
5746	* fixed bits. We thus need to re-apply them while importing CR4.
5747	*/
5748	u64GuestCr4 \|= fSetCr4;
5749	u64GuestCr4 &= fZapCr4;
5750
5751	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5752	/** @todo Fix to 64-bit when we drop 32-bit. */
5753	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5754	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5755	AssertRCReturn(rc, rc);
5756
5757	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5758	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5759
5760	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5761
5762	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5763	}
5764	return rc;
5765	}
5766
5767
5768	/**
5769	* Exports the guest debug registers into the guest-state area in the VMCS.
5770	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5771	*
5772	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5773	*
5774	* @returns VBox status code.
5775	* @param pVCpu The cross context virtual CPU structure.
5776	* @param pVmxTransient The VMX-transient structure.
5777	*
5778	* @remarks No-long-jump zone!!!
5779	*/
5780	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5781	{
5782	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5783
5784	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5785	* stepping. */
5786	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5787	if (pVmxTransient->fIsNestedGuest)
5788	{
5789	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5790	AssertRCReturn(rc, rc);
5791	return VINF_SUCCESS;
5792	}
5793
5794	#ifdef VBOX_STRICT
5795	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5796	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5797	{
5798	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5799	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5800	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5801	}
5802	#endif
5803
5804	bool fSteppingDB = false;
5805	bool fInterceptMovDRx = false;
5806	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5807	if (pVCpu->hm.s.fSingleInstruction)
5808	{
5809	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5810	PVM pVM = pVCpu->CTX_SUFF(pVM);
5811	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5812	{
5813	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5814	Assert(fSteppingDB == false);
5815	}
5816	else
5817	{
5818	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5819	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5820	pVCpu->hm.s.fClearTrapFlag = true;
5821	fSteppingDB = true;
5822	}
5823	}
5824
5825	uint32_t u32GuestDr7;
5826	if ( fSteppingDB
5827	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5828	{
5829	/*
5830	* Use the combined guest and host DRx values found in the hypervisor register set
5831	* because the hypervisor debugger has breakpoints active or someone is single stepping
5832	* on the host side without a monitor trap flag.
5833	*
5834	* Note! DBGF expects a clean DR6 state before executing guest code.
5835	*/
5836	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5837	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5838	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5839	{
5840	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5841	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5842	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5843	}
5844	else
5845	#endif
5846	if (!CPUMIsHyperDebugStateActive(pVCpu))
5847	{
5848	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5849	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5850	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5851	}
5852
5853	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5854	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5855	pVCpu->hm.s.fUsingHyperDR7 = true;
5856	fInterceptMovDRx = true;
5857	}
5858	else
5859	{
5860	/*
5861	* If the guest has enabled debug registers, we need to load them prior to
5862	* executing guest code so they'll trigger at the right time.
5863	*/
5864	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5865	{
5866	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5867	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5868	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5869	{
5870	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5871	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5872	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5873	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5874	}
5875	else
5876	#endif
5877	if (!CPUMIsGuestDebugStateActive(pVCpu))
5878	{
5879	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5880	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5881	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5882	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5883	}
5884	Assert(!fInterceptMovDRx);
5885	}
5886	/*
5887	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5888	* must intercept #DB in order to maintain a correct DR6 guest value, and
5889	* because we need to intercept it to prevent nested #DBs from hanging the
5890	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5891	*/
5892	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5893	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5894	&& !CPUMIsGuestDebugStateActive(pVCpu))
5895	#else
5896	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5897	#endif
5898	{
5899	fInterceptMovDRx = true;
5900	}
5901
5902	/* Update DR7 with the actual guest value. */
5903	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5904	pVCpu->hm.s.fUsingHyperDR7 = false;
5905	}
5906
5907	if (fInterceptMovDRx)
5908	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5909	else
5910	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5911
5912	/*
5913	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5914	* monitor-trap flag and update our cache.
5915	*/
5916	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5917	{
5918	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5919	AssertRCReturn(rc2, rc2);
5920	pVmcsInfo->u32ProcCtls = uProcCtls;
5921	}
5922
5923	/*
5924	* Update guest DR7.
5925	*/
5926	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5927	AssertRCReturn(rc, rc);
5928
5929	/*
5930	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5931	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5932	*
5933	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5934	*/
5935	if (fSteppingDB)
5936	{
5937	Assert(pVCpu->hm.s.fSingleInstruction);
5938	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5939
5940	uint32_t fIntrState = 0;
5941	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5942	AssertRCReturn(rc, rc);
5943
5944	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5945	{
5946	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5947	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5948	AssertRCReturn(rc, rc);
5949	}
5950	}
5951
5952	return VINF_SUCCESS;
5953	}
5954
5955
5956	#ifdef VBOX_STRICT
5957	/**
5958	* Strict function to validate segment registers.
5959	*
5960	* @param pVCpu The cross context virtual CPU structure.
5961	* @param pVmcsInfo The VMCS info. object.
5962	*
5963	* @remarks Will import guest CR0 on strict builds during validation of
5964	* segments.
5965	*/
5966	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5967	{
5968	/*
5969	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5970	*
5971	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5972	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5973	* unusable bit and doesn't change the guest-context value.
5974	*/
5975	PVM pVM = pVCpu->CTX_SUFF(pVM);
5976	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5977	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5978	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5979	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5980	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5981	{
5982	/* Protected mode checks */
5983	/* CS */
5984	Assert(pCtx->cs.Attr.n.u1Present);
5985	Assert(!(pCtx->cs.Attr.u & 0xf00));
5986	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5987	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5988	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5989	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5990	\|\| (pCtx->cs.Attr.n.u1Granularity));
5991	/* CS cannot be loaded with NULL in protected mode. */
5992	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5993	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5994	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5995	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5996	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5997	else
5998	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5999	/* SS */
6000	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6001	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6002	if ( !(pCtx->cr0 & X86_CR0_PE)
6003	\|\| pCtx->cs.Attr.n.u4Type == 3)
6004	{
6005	Assert(!pCtx->ss.Attr.n.u2Dpl);
6006	}
6007	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6008	{
6009	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6010	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6011	Assert(pCtx->ss.Attr.n.u1Present);
6012	Assert(!(pCtx->ss.Attr.u & 0xf00));
6013	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6014	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6015	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6016	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6017	\|\| (pCtx->ss.Attr.n.u1Granularity));
6018	}
6019	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6020	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6021	{
6022	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6023	Assert(pCtx->ds.Attr.n.u1Present);
6024	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6025	Assert(!(pCtx->ds.Attr.u & 0xf00));
6026	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6027	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6028	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6029	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6030	\|\| (pCtx->ds.Attr.n.u1Granularity));
6031	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6032	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6033	}
6034	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6035	{
6036	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6037	Assert(pCtx->es.Attr.n.u1Present);
6038	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6039	Assert(!(pCtx->es.Attr.u & 0xf00));
6040	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6041	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6042	\|\| !(pCtx->es.Attr.n.u1Granularity));
6043	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6044	\|\| (pCtx->es.Attr.n.u1Granularity));
6045	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6046	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6047	}
6048	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6049	{
6050	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6051	Assert(pCtx->fs.Attr.n.u1Present);
6052	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6053	Assert(!(pCtx->fs.Attr.u & 0xf00));
6054	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6055	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6056	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6057	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6058	\|\| (pCtx->fs.Attr.n.u1Granularity));
6059	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6060	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6061	}
6062	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6063	{
6064	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6065	Assert(pCtx->gs.Attr.n.u1Present);
6066	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6067	Assert(!(pCtx->gs.Attr.u & 0xf00));
6068	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6069	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6070	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6071	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6072	\|\| (pCtx->gs.Attr.n.u1Granularity));
6073	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6074	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6075	}
6076	/* 64-bit capable CPUs. */
6077	# if HC_ARCH_BITS == 64
6078	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6079	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6080	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6081	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6082	# endif
6083	}
6084	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6085	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6086	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6087	{
6088	/* Real and v86 mode checks. */
6089	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6090	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6091	if (pVmcsInfo->RealMode.fRealOnV86Active)
6092	{
6093	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6094	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6095	}
6096	else
6097	{
6098	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6099	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6100	}
6101
6102	/* CS */
6103	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6104	Assert(pCtx->cs.u32Limit == 0xffff);
6105	Assert(u32CSAttr == 0xf3);
6106	/* SS */
6107	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6108	Assert(pCtx->ss.u32Limit == 0xffff);
6109	Assert(u32SSAttr == 0xf3);
6110	/* DS */
6111	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6112	Assert(pCtx->ds.u32Limit == 0xffff);
6113	Assert(u32DSAttr == 0xf3);
6114	/* ES */
6115	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6116	Assert(pCtx->es.u32Limit == 0xffff);
6117	Assert(u32ESAttr == 0xf3);
6118	/* FS */
6119	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6120	Assert(pCtx->fs.u32Limit == 0xffff);
6121	Assert(u32FSAttr == 0xf3);
6122	/* GS */
6123	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6124	Assert(pCtx->gs.u32Limit == 0xffff);
6125	Assert(u32GSAttr == 0xf3);
6126	/* 64-bit capable CPUs. */
6127	# if HC_ARCH_BITS == 64
6128	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6129	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6130	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6131	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6132	# endif
6133	}
6134	}
6135	#endif /* VBOX_STRICT */
6136
6137
6138	/**
6139	* Exports a guest segment register into the guest-state area in the VMCS.
6140	*
6141	* @returns VBox status code.
6142	* @param pVCpu The cross context virtual CPU structure.
6143	* @param pVmcsInfo The VMCS info. object.
6144	* @param iSegReg The segment register number (X86_SREG_XXX).
6145	* @param pSelReg Pointer to the segment selector.
6146	*
6147	* @remarks No-long-jump zone!!!
6148	*/
6149	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6150	{
6151	Assert(iSegReg < X86_SREG_COUNT);
6152	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6153	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6154	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6155	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6156
6157	uint32_t u32Access = pSelReg->Attr.u;
6158	if (pVmcsInfo->RealMode.fRealOnV86Active)
6159	{
6160	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6161	u32Access = 0xf3;
6162	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6163	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6164	RT_NOREF_PV(pVCpu);
6165	}
6166	else
6167	{
6168	/*
6169	* The way to differentiate between whether this is really a null selector or was just
6170	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6171	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6172	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6173	* NULL selectors loaded in protected-mode have their attribute as 0.
6174	*/
6175	if (!u32Access)
6176	u32Access = X86DESCATTR_UNUSABLE;
6177	}
6178
6179	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6180	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6181	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6182
6183	/*
6184	* Commit it to the VMCS.
6185	*/
6186	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6187	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6188	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6189	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6190	AssertRCReturn(rc, rc);
6191	return rc;
6192	}
6193
6194
6195	/**
6196	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6197	* area in the VMCS.
6198	*
6199	* @returns VBox status code.
6200	* @param pVCpu The cross context virtual CPU structure.
6201	* @param pVmxTransient The VMX-transient structure.
6202	*
6203	* @remarks Will import guest CR0 on strict builds during validation of
6204	* segments.
6205	* @remarks No-long-jump zone!!!
6206	*/
6207	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6208	{
6209	int rc = VERR_INTERNAL_ERROR_5;
6210	PVM pVM = pVCpu->CTX_SUFF(pVM);
6211	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6212	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6213
6214	/*
6215	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6216	*/
6217	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6218	{
6219	#ifdef VBOX_WITH_REM
6220	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6221	{
6222	Assert(!pVmxTransient->fIsNestedGuest);
6223	Assert(pVM->hm.s.vmx.pRealModeTSS);
6224	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6225	if ( pVmcsInfo->fWasInRealMode
6226	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6227	{
6228	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
6229	in real-mode (e.g. OpenBSD 4.0) */
6230	REMFlushTBs(pVM);
6231	Log4Func(("Switch to protected mode detected!\n"));
6232	pVmcsInfo->fWasInRealMode = false;
6233	}
6234	}
6235	#endif
6236	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6237	{
6238	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6239	if (pVmcsInfo->RealMode.fRealOnV86Active)
6240	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6241	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6242	AssertRCReturn(rc, rc);
6243	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6244	}
6245
6246	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6247	{
6248	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6249	if (pVmcsInfo->RealMode.fRealOnV86Active)
6250	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6251	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6252	AssertRCReturn(rc, rc);
6253	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6254	}
6255
6256	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6257	{
6258	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6259	if (pVmcsInfo->RealMode.fRealOnV86Active)
6260	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6261	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6262	AssertRCReturn(rc, rc);
6263	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6264	}
6265
6266	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6267	{
6268	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6269	if (pVmcsInfo->RealMode.fRealOnV86Active)
6270	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6271	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6272	AssertRCReturn(rc, rc);
6273	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6274	}
6275
6276	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6277	{
6278	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6279	if (pVmcsInfo->RealMode.fRealOnV86Active)
6280	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6281	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6282	AssertRCReturn(rc, rc);
6283	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6284	}
6285
6286	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6287	{
6288	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6289	if (pVmcsInfo->RealMode.fRealOnV86Active)
6290	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6291	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6292	AssertRCReturn(rc, rc);
6293	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6294	}
6295
6296	#ifdef VBOX_STRICT
6297	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6298	#endif
6299	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6300	pCtx->cs.Attr.u));
6301	}
6302
6303	/*
6304	* Guest TR.
6305	*/
6306	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6307	{
6308	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6309
6310	/*
6311	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6312	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6313	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6314	*/
6315	uint16_t u16Sel;
6316	uint32_t u32Limit;
6317	uint64_t u64Base;
6318	uint32_t u32AccessRights;
6319	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6320	{
6321	u16Sel = pCtx->tr.Sel;
6322	u32Limit = pCtx->tr.u32Limit;
6323	u64Base = pCtx->tr.u64Base;
6324	u32AccessRights = pCtx->tr.Attr.u;
6325	}
6326	else
6327	{
6328	Assert(!pVmxTransient->fIsNestedGuest);
6329	Assert(pVM->hm.s.vmx.pRealModeTSS);
6330	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6331
6332	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6333	RTGCPHYS GCPhys;
6334	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6335	AssertRCReturn(rc, rc);
6336
6337	X86DESCATTR DescAttr;
6338	DescAttr.u = 0;
6339	DescAttr.n.u1Present = 1;
6340	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6341
6342	u16Sel = 0;
6343	u32Limit = HM_VTX_TSS_SIZE;
6344	u64Base = GCPhys;
6345	u32AccessRights = DescAttr.u;
6346	}
6347
6348	/* Validate. */
6349	Assert(!(u16Sel & RT_BIT(2)));
6350	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6351	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6352	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6353	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6354	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6355	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6356	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6357	Assert( (u32Limit & 0xfff) == 0xfff
6358	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6359	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6360	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6361
6362	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6363	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6364	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6365	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6366	AssertRCReturn(rc, rc);
6367
6368	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6369	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6370	}
6371
6372	/*
6373	* Guest GDTR.
6374	*/
6375	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6376	{
6377	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6378
6379	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6380	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6381	AssertRCReturn(rc, rc);
6382
6383	/* Validate. */
6384	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6385
6386	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6387	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6388	}
6389
6390	/*
6391	* Guest LDTR.
6392	*/
6393	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6394	{
6395	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6396
6397	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6398	uint32_t u32Access;
6399	if ( !pVmxTransient->fIsNestedGuest
6400	&& !pCtx->ldtr.Attr.u)
6401	u32Access = X86DESCATTR_UNUSABLE;
6402	else
6403	u32Access = pCtx->ldtr.Attr.u;
6404
6405	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6406	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6407	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6408	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6409	AssertRCReturn(rc, rc);
6410
6411	/* Validate. */
6412	if (!(u32Access & X86DESCATTR_UNUSABLE))
6413	{
6414	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6415	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6416	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6417	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6418	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6419	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6420	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6421	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6422	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6423	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6424	}
6425
6426	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6427	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6428	}
6429
6430	/*
6431	* Guest IDTR.
6432	*/
6433	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6434	{
6435	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6436
6437	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6438	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6439	AssertRCReturn(rc, rc);
6440
6441	/* Validate. */
6442	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6443
6444	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6445	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6446	}
6447
6448	return VINF_SUCCESS;
6449	}
6450
6451
6452	/**
6453	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6454	* areas.
6455	*
6456	* These MSRs will automatically be loaded to the host CPU on every successful
6457	* VM-entry and stored from the host CPU on every successful VM-exit.
6458	*
6459	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6460	* actual host MSR values are not- updated here for performance reasons. See
6461	* hmR0VmxExportHostMsrs().
6462	*
6463	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6464	*
6465	* @returns VBox status code.
6466	* @param pVCpu The cross context virtual CPU structure.
6467	* @param pVmxTransient The VMX-transient structure.
6468	*
6469	* @remarks No-long-jump zone!!!
6470	*/
6471	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6472	{
6473	AssertPtr(pVCpu);
6474	AssertPtr(pVmxTransient);
6475
6476	PVM pVM = pVCpu->CTX_SUFF(pVM);
6477	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6478
6479	/*
6480	* MSRs that we use the auto-load/store MSR area in the VMCS.
6481	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6482	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6483	*
6484	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6485	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6486	* emulation, nothing to do here.
6487	*/
6488	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6489	{
6490	if ( !pVmxTransient->fIsNestedGuest
6491	&& pVM->hm.s.fAllow64BitGuests)
6492	{
6493	#if HC_ARCH_BITS == 32
6494	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6495	Assert(!pVmxTransient->fIsNestedGuest);
6496
6497	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6498	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6499	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6500	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6501	AssertRCReturn(rc, rc);
6502	#endif
6503	}
6504	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6505	}
6506
6507	/*
6508	* Guest Sysenter MSRs.
6509	*/
6510	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6511	{
6512	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6513
6514	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6515	{
6516	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6517	AssertRCReturn(rc, rc);
6518	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6519	}
6520
6521	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6522	{
6523	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6524	AssertRCReturn(rc, rc);
6525	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6526	}
6527
6528	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6529	{
6530	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6531	AssertRCReturn(rc, rc);
6532	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6533	}
6534	}
6535
6536	/*
6537	* Guest/host EFER MSR.
6538	*/
6539	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6540	{
6541	/* Whether we are using the VMCS to swap the EFER MSR must have been
6542	determined earlier while exporting VM-entry/VM-exit controls. */
6543	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6544	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6545
6546	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6547	{
6548	/*
6549	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6550	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6551	*/
6552	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6553	{
6554	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6555	AssertRCReturn(rc, rc);
6556	}
6557	else
6558	{
6559	/*
6560	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6561	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6562	*/
6563	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6564	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6565	AssertRCReturn(rc, rc);
6566	}
6567	}
6568	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6569	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6570
6571	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6572	}
6573
6574	/*
6575	* Other MSRs.
6576	* Speculation Control (R/W).
6577	*/
6578	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6579	{
6580	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6581	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6582	{
6583	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6584	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6585	AssertRCReturn(rc, rc);
6586	}
6587	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6588	}
6589
6590	return VINF_SUCCESS;
6591	}
6592
6593
6594	/**
6595	* Selects up the appropriate function to run guest code.
6596	*
6597	* @returns VBox status code.
6598	* @param pVCpu The cross context virtual CPU structure.
6599	* @param pVmxTransient The VMX-transient structure.
6600	*
6601	* @remarks No-long-jump zone!!!
6602	*/
6603	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6604	{
6605	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6606	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6607
6608	if (CPUMIsGuestInLongModeEx(pCtx))
6609	{
6610	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6611	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6612	#endif
6613	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6614	#if HC_ARCH_BITS == 32
6615	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6616	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6617	{
6618	#ifdef VBOX_STRICT
6619	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6620	{
6621	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6622	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6623	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6624	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6625	("fCtxChanged=%#RX64\n", fCtxChanged));
6626	}
6627	#endif
6628	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6629
6630	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6631	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6632	pVmcsInfo->fSwitchedTo64on32 = true;
6633	Log4Func(("Selected 64-bit switcher\n"));
6634	}
6635	#else
6636	/* 64-bit host. */
6637	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6638	#endif
6639	}
6640	else
6641	{
6642	/* Guest is not in long mode, use the 32-bit handler. */
6643	#if HC_ARCH_BITS == 32
6644	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6645	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6646	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6647	{
6648	# ifdef VBOX_STRICT
6649	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6650	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6651	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6652	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6653	("fCtxChanged=%#RX64\n", fCtxChanged));
6654	# endif
6655	}
6656	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6657	/*
6658	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6659	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6660	* switcher flag now because we know the guest is in a sane state where it's safe
6661	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6662	* the much faster 32-bit switcher again.
6663	*/
6664	if (!pVmcsInfo->fSwitchedTo64on32)
6665	{
6666	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6667	Log4Func(("Selected 32-bit switcher\n"));
6668	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6669	}
6670	else
6671	{
6672	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6673	if ( pVmcsInfo->RealMode.fRealOnV86Active
6674	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6675	{
6676	pVmcsInfo->fSwitchedTo64on32 = false;
6677	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6678	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6679	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6680	\| HM_CHANGED_HOST_CONTEXT);
6681	Log4Func(("Selected 32-bit switcher (safe)\n"));
6682	}
6683	}
6684	# else
6685	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6686	# endif
6687	#else
6688	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6689	#endif
6690	}
6691	Assert(pVmcsInfo->pfnStartVM);
6692	return VINF_SUCCESS;
6693	}
6694
6695
6696	/**
6697	* Wrapper for running the guest code in VT-x.
6698	*
6699	* @returns VBox status code, no informational status codes.
6700	* @param pVCpu The cross context virtual CPU structure.
6701	* @param pVmxTransient The VMX-transient structure.
6702	*
6703	* @remarks No-long-jump zone!!!
6704	*/
6705	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6706	{
6707	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6708	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6709	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6710
6711	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6712
6713	/*
6714	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6715	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6716	* callee-saved and thus the need for this XMM wrapper.
6717	*
6718	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6719	*/
6720	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6721	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6722	PVM pVM = pVCpu->CTX_SUFF(pVM);
6723	#ifdef VBOX_WITH_KERNEL_USING_XMM
6724	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6725	#else
6726	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6727	#endif
6728	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6729	return rc;
6730	}
6731
6732
6733	/**
6734	* Reports world-switch error and dumps some useful debug info.
6735	*
6736	* @param pVCpu The cross context virtual CPU structure.
6737	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6738	* @param pVmxTransient The VMX-transient structure (only
6739	* exitReason updated).
6740	*/
6741	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6742	{
6743	Assert(pVCpu);
6744	Assert(pVmxTransient);
6745	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6746
6747	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6748	switch (rcVMRun)
6749	{
6750	case VERR_VMX_INVALID_VMXON_PTR:
6751	AssertFailed();
6752	break;
6753	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6754	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6755	{
6756	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6757	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6758	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6759	AssertRC(rc);
6760
6761	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6762	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6763	Cannot do it here as we may have been long preempted. */
6764
6765	#ifdef VBOX_STRICT
6766	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6767	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6768	pVmxTransient->uExitReason));
6769	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6770	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6771	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6772	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6773	else
6774	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6775	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6776	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6777
6778	/* VMX control bits. */
6779	uint32_t u32Val;
6780	uint64_t u64Val;
6781	RTHCUINTREG uHCReg;
6782	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6783	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6784	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6785	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6786	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6787	{
6788	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6789	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6790	}
6791	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6792	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6793	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6794	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6795	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6796	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6797	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6798	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6799	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6800	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6801	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6802	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6803	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6804	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6805	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6806	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6807	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6808	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6809	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6810	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6811	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6812	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6814	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6816	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6817	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6818	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6819	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6820	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6821	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6822	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6823	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6824	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6825	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6826	{
6827	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6828	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6829	}
6830
6831	/* Guest bits. */
6832	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6833	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6834	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6835	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6836	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6837	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6838	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6839	{
6840	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6841	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6842	}
6843
6844	/* Host bits. */
6845	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6846	Log4(("Host CR0 %#RHr\n", uHCReg));
6847	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6848	Log4(("Host CR3 %#RHr\n", uHCReg));
6849	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6850	Log4(("Host CR4 %#RHr\n", uHCReg));
6851
6852	RTGDTR HostGdtr;
6853	PCX86DESCHC pDesc;
6854	ASMGetGDTR(&HostGdtr);
6855	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6856	Log4(("Host CS %#08x\n", u32Val));
6857	if (u32Val < HostGdtr.cbGdt)
6858	{
6859	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6860	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6861	}
6862
6863	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6864	Log4(("Host DS %#08x\n", u32Val));
6865	if (u32Val < HostGdtr.cbGdt)
6866	{
6867	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6868	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6869	}
6870
6871	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6872	Log4(("Host ES %#08x\n", u32Val));
6873	if (u32Val < HostGdtr.cbGdt)
6874	{
6875	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6876	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6877	}
6878
6879	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6880	Log4(("Host FS %#08x\n", u32Val));
6881	if (u32Val < HostGdtr.cbGdt)
6882	{
6883	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6884	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6885	}
6886
6887	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6888	Log4(("Host GS %#08x\n", u32Val));
6889	if (u32Val < HostGdtr.cbGdt)
6890	{
6891	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6892	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6893	}
6894
6895	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6896	Log4(("Host SS %#08x\n", u32Val));
6897	if (u32Val < HostGdtr.cbGdt)
6898	{
6899	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6900	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6901	}
6902
6903	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6904	Log4(("Host TR %#08x\n", u32Val));
6905	if (u32Val < HostGdtr.cbGdt)
6906	{
6907	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6908	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6909	}
6910
6911	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6912	Log4(("Host TR Base %#RHv\n", uHCReg));
6913	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6914	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6915	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6916	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6917	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6918	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6919	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6920	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6921	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6922	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6923	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6924	Log4(("Host RSP %#RHv\n", uHCReg));
6925	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6926	Log4(("Host RIP %#RHv\n", uHCReg));
6927	# if HC_ARCH_BITS == 64
6928	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6929	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6930	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6931	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6932	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6933	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6934	# endif
6935	#endif /* VBOX_STRICT */
6936	break;
6937	}
6938
6939	default:
6940	/* Impossible */
6941	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6942	break;
6943	}
6944	}
6945
6946
6947	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6948	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6949	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6950	# endif
6951
6952	/**
6953	* Initialize the VMCS-Read cache.
6954	*
6955	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6956	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6957	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6958	* (those that have a 32-bit FULL & HIGH part).
6959	*
6960	* @param pVCpu The cross context virtual CPU structure.
6961	*/
6962	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6963	{
6964	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6965	do { \
6966	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6967	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6968	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6969	++cReadFields; \
6970	} while (0)
6971
6972	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6973	uint32_t cReadFields = 0;
6974
6975	/*
6976	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6977	* and serve to indicate exceptions to the rules.
6978	*/
6979
6980	/* Guest-natural selector base fields. */
6981	#if 0
6982	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6983	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6984	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6985	#endif
6986	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6987	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6988	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6989	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6990	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6991	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6992	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6993	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6994	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6995	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6996	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6997	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6998	#if 0
6999	/* Unused natural width guest-state fields. */
7000	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7001	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7002	#endif
7003	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7004	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7005
7006	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7007	these 64-bit fields (using "FULL" and "HIGH" fields). */
7008	#if 0
7009	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7010	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7011	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7012	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7013	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7014	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7015	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7016	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7017	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7018	#endif
7019
7020	/* Natural width guest-state fields. */
7021	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7022	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7023
7024	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7025	{
7026	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7027	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7028	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7029	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7030	}
7031	else
7032	{
7033	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7034	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7035	}
7036
7037	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7038	}
7039
7040
7041	/**
7042	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7043	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7044	* darwin, running 64-bit guests).
7045	*
7046	* @returns VBox status code.
7047	* @param pVCpu The cross context virtual CPU structure.
7048	* @param idxField The VMCS field encoding.
7049	* @param u64Val 16, 32 or 64-bit value.
7050	*/
7051	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7052	{
7053	AssertPtr(pVCpu);
7054	int rc;
7055	switch (idxField)
7056	{
7057	/*
7058	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7059	*/
7060	/* 64-bit Control fields. */
7061	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7062	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7063	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7064	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7065	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7066	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7067	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7068	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7069	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7070	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7071	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7072	case VMX_VMCS64_CTRL_EPTP_FULL:
7073	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7074	/* 64-bit Guest-state fields. */
7075	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7076	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7077	case VMX_VMCS64_GUEST_PAT_FULL:
7078	case VMX_VMCS64_GUEST_EFER_FULL:
7079	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7080	case VMX_VMCS64_GUEST_PDPTE0_FULL:
7081	case VMX_VMCS64_GUEST_PDPTE1_FULL:
7082	case VMX_VMCS64_GUEST_PDPTE2_FULL:
7083	case VMX_VMCS64_GUEST_PDPTE3_FULL:
7084	/* 64-bit Host-state fields. */
7085	case VMX_VMCS64_HOST_PAT_FULL:
7086	case VMX_VMCS64_HOST_EFER_FULL:
7087	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7088	{
7089	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7090	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7091	break;
7092	}
7093
7094	/*
7095	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7096	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7097	*/
7098	/* Natural-width Guest-state fields. */
7099	case VMX_VMCS_GUEST_CR3:
7100	case VMX_VMCS_GUEST_ES_BASE:
7101	case VMX_VMCS_GUEST_CS_BASE:
7102	case VMX_VMCS_GUEST_SS_BASE:
7103	case VMX_VMCS_GUEST_DS_BASE:
7104	case VMX_VMCS_GUEST_FS_BASE:
7105	case VMX_VMCS_GUEST_GS_BASE:
7106	case VMX_VMCS_GUEST_LDTR_BASE:
7107	case VMX_VMCS_GUEST_TR_BASE:
7108	case VMX_VMCS_GUEST_GDTR_BASE:
7109	case VMX_VMCS_GUEST_IDTR_BASE:
7110	case VMX_VMCS_GUEST_RSP:
7111	case VMX_VMCS_GUEST_RIP:
7112	case VMX_VMCS_GUEST_SYSENTER_ESP:
7113	case VMX_VMCS_GUEST_SYSENTER_EIP:
7114	{
7115	if (!(RT_HI_U32(u64Val)))
7116	{
7117	/* If this field is 64-bit, VT-x will zero out the top bits. */
7118	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7119	}
7120	else
7121	{
7122	/* Assert that only the 32->64 switcher case should ever come here. */
7123	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7124	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7125	}
7126	break;
7127	}
7128
7129	default:
7130	{
7131	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7132	pVCpu->hm.s.u32HMError = idxField;
7133	rc = VERR_INVALID_PARAMETER;
7134	break;
7135	}
7136	}
7137	AssertRCReturn(rc, rc);
7138	return rc;
7139	}
7140
7141
7142	/**
7143	* Queue up a VMWRITE by using the VMCS write cache.
7144	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7145	*
7146	* @param pVCpu The cross context virtual CPU structure.
7147	* @param idxField The VMCS field encoding.
7148	* @param u64Val 16, 32 or 64-bit value.
7149	*/
7150	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7151	{
7152	AssertPtr(pVCpu);
7153	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7154
7155	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7156	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7157
7158	/* Make sure there are no duplicates. */
7159	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7160	{
7161	if (pCache->Write.aField[i] == idxField)
7162	{
7163	pCache->Write.aFieldVal[i] = u64Val;
7164	return VINF_SUCCESS;
7165	}
7166	}
7167
7168	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7169	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7170	pCache->Write.cValidEntries++;
7171	return VINF_SUCCESS;
7172	}
7173	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7174
7175
7176	/**
7177	* Sets up the usage of TSC-offsetting and updates the VMCS.
7178	*
7179	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7180	* VMX-preemption timer.
7181	*
7182	* @returns VBox status code.
7183	* @param pVCpu The cross context virtual CPU structure.
7184	* @param pVmxTransient The VMX-transient structure.
7185	*
7186	* @remarks No-long-jump zone!!!
7187	*/
7188	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7189	{
7190	bool fOffsettedTsc;
7191	bool fParavirtTsc;
7192	uint64_t uTscOffset;
7193	PVM pVM = pVCpu->CTX_SUFF(pVM);
7194	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7195
7196	if (pVM->hm.s.vmx.fUsePreemptTimer)
7197	{
7198	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7199
7200	/* Make sure the returned values have sane upper and lower boundaries. */
7201	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7202	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7203	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7204	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7205
7206	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7207	* preemption timers here. We probably need to clamp the preemption timer,
7208	* after converting the timer value to the host. */
7209	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7210	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7211	AssertRC(rc);
7212	}
7213	else
7214	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7215
7216	if (fParavirtTsc)
7217	{
7218	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7219	information before every VM-entry, hence disable it for performance sake. */
7220	#if 0
7221	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7222	AssertRC(rc);
7223	#endif
7224	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7225	}
7226
7227	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
7228	if ( fOffsettedTsc
7229	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7230	{
7231	if (pVmxTransient->fIsNestedGuest)
7232	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7233	if (pVmcsInfo->u64TscOffset != uTscOffset)
7234	{
7235	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
7236	AssertRC(rc);
7237	pVmcsInfo->u64TscOffset = uTscOffset;
7238	}
7239
7240	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
7241	{
7242	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
7243	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7244	AssertRC(rc);
7245	pVmcsInfo->u32ProcCtls = uProcCtls;
7246	}
7247	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
7248	}
7249	else
7250	{
7251	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7252	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
7253	{
7254	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
7255	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7256	AssertRC(rc);
7257	pVmcsInfo->u32ProcCtls = uProcCtls;
7258	}
7259	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
7260	}
7261	}
7262
7263
7264	/**
7265	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7266	* VM-exit interruption info type.
7267	*
7268	* @returns The IEM exception flags.
7269	* @param uVector The event vector.
7270	* @param uVmxEventType The VMX event type.
7271	*
7272	* @remarks This function currently only constructs flags required for
7273	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7274	* and CR2 aspects of an exception are not included).
7275	*/
7276	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7277	{
7278	uint32_t fIemXcptFlags;
7279	switch (uVmxEventType)
7280	{
7281	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7282	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7283	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7284	break;
7285
7286	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7287	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7288	break;
7289
7290	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7291	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7292	break;
7293
7294	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7295	{
7296	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7297	if (uVector == X86_XCPT_BP)
7298	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7299	else if (uVector == X86_XCPT_OF)
7300	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7301	else
7302	{
7303	fIemXcptFlags = 0;
7304	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7305	}
7306	break;
7307	}
7308
7309	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7310	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7311	break;
7312
7313	default:
7314	fIemXcptFlags = 0;
7315	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7316	break;
7317	}
7318	return fIemXcptFlags;
7319	}
7320
7321
7322	/**
7323	* Sets an event as a pending event to be injected into the guest.
7324	*
7325	* @param pVCpu The cross context virtual CPU structure.
7326	* @param u32IntInfo The VM-entry interruption-information field.
7327	* @param cbInstr The VM-entry instruction length in bytes (for software
7328	* interrupts, exceptions and privileged software
7329	* exceptions).
7330	* @param u32ErrCode The VM-entry exception error code.
7331	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7332	* page-fault.
7333	*/
7334	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7335	RTGCUINTPTR GCPtrFaultAddress)
7336	{
7337	Assert(!pVCpu->hm.s.Event.fPending);
7338	pVCpu->hm.s.Event.fPending = true;
7339	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7340	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7341	pVCpu->hm.s.Event.cbInstr = cbInstr;
7342	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7343	}
7344
7345
7346	/**
7347	* Sets an external interrupt as pending-for-injection into the VM.
7348	*
7349	* @param pVCpu The cross context virtual CPU structure.
7350	* @param u8Interrupt The external interrupt vector.
7351	*/
7352	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7353	{
7354	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7355	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7356	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7357	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7358	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7359	}
7360
7361
7362	/**
7363	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7364	*
7365	* @param pVCpu The cross context virtual CPU structure.
7366	*/
7367	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7368	{
7369	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7370	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7371	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7372	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7373	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7374	}
7375
7376
7377	/**
7378	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7379	*
7380	* @param pVCpu The cross context virtual CPU structure.
7381	*/
7382	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7383	{
7384	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7385	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7386	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7387	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7388	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7389	}
7390
7391
7392	/**
7393	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7394	*
7395	* @param pVCpu The cross context virtual CPU structure.
7396	*/
7397	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7398	{
7399	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7400	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7401	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7402	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7403	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7404	}
7405
7406
7407	/**
7408	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7409	*
7410	* @param pVCpu The cross context virtual CPU structure.
7411	*/
7412	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7413	{
7414	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7415	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7416	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7417	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7418	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7419	}
7420
7421
7422	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7423	/**
7424	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7425	*
7426	* @param pVCpu The cross context virtual CPU structure.
7427	* @param u32ErrCode The error code for the general-protection exception.
7428	*/
7429	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7430	{
7431	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7432	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7433	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7434	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7435	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7436	}
7437
7438
7439	/**
7440	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7441	*
7442	* @param pVCpu The cross context virtual CPU structure.
7443	* @param u32ErrCode The error code for the stack exception.
7444	*/
7445	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7446	{
7447	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7448	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7449	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7450	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7451	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7452	}
7453
7454
7455	/**
7456	* Decodes the memory operand of an instruction that caused a VM-exit.
7457	*
7458	* The Exit qualification field provides the displacement field for memory
7459	* operand instructions, if any.
7460	*
7461	* @returns Strict VBox status code (i.e. informational status codes too).
7462	* @retval VINF_SUCCESS if the operand was successfully decoded.
7463	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
7464	* operand.
7465	* @param pVCpu The cross context virtual CPU structure.
7466	* @param uExitInstrInfo The VM-exit instruction information field.
7467	* @param enmMemAccess The memory operand's access type (read or write).
7468	* @param GCPtrDisp The instruction displacement field, if any. For
7469	* RIP-relative addressing pass RIP + displacement here.
7470	* @param pGCPtrMem Where to store the effective destination memory address.
7471	*
7472	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
7473	* virtual-8086 mode hence skips those checks while verifying if the
7474	* segment is valid.
7475	*/
7476	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
7477	PRTGCPTR pGCPtrMem)
7478	{
7479	Assert(pGCPtrMem);
7480	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
7481	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
7482	\| CPUMCTX_EXTRN_CR0);
7483
7484	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
7485	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
7486	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
7487
7488	VMXEXITINSTRINFO ExitInstrInfo;
7489	ExitInstrInfo.u = uExitInstrInfo;
7490	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
7491	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
7492	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
7493	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
7494	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
7495	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
7496	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
7497	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
7498	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
7499
7500	/*
7501	* Validate instruction information.
7502	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
7503	*/
7504	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
7505	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
7506	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
7507	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
7508	AssertLogRelMsgReturn(fIsMemOperand,
7509	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
7510
7511	/*
7512	* Compute the complete effective address.
7513	*
7514	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
7515	* See AMD spec. 4.5.2 "Segment Registers".
7516	*/
7517	RTGCPTR GCPtrMem = GCPtrDisp;
7518	if (fBaseRegValid)
7519	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
7520	if (fIdxRegValid)
7521	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
7522
7523	RTGCPTR const GCPtrOff = GCPtrMem;
7524	if ( !fIsLongMode
7525	\|\| iSegReg >= X86_SREG_FS)
7526	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
7527	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
7528
7529	/*
7530	* Validate effective address.
7531	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
7532	*/
7533	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
7534	Assert(cbAccess > 0);
7535	if (fIsLongMode)
7536	{
7537	if (X86_IS_CANONICAL(GCPtrMem))
7538	{
7539	*pGCPtrMem = GCPtrMem;
7540	return VINF_SUCCESS;
7541	}
7542
7543	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
7544	* "Data Limit Checks in 64-bit Mode". */
7545	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
7546	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7547	return VINF_HM_PENDING_XCPT;
7548	}
7549
7550	/*
7551	* This is a watered down version of iemMemApplySegment().
7552	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
7553	* and segment CPL/DPL checks are skipped.
7554	*/
7555	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
7556	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
7557	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7558
7559	/* Check if the segment is present and usable. */
7560	if ( pSel->Attr.n.u1Present
7561	&& !pSel->Attr.n.u1Unusable)
7562	{
7563	Assert(pSel->Attr.n.u1DescType);
7564	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
7565	{
7566	/* Check permissions for the data segment. */
7567	if ( enmMemAccess == VMXMEMACCESS_WRITE
7568	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
7569	{
7570	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7571	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
7572	return VINF_HM_PENDING_XCPT;
7573	}
7574
7575	/* Check limits if it's a normal data segment. */
7576	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
7577	{
7578	if ( GCPtrFirst32 > pSel->u32Limit
7579	\|\| GCPtrLast32 > pSel->u32Limit)
7580	{
7581	Log4Func(("Data segment limit exceeded. "
7582	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7583	GCPtrLast32, pSel->u32Limit));
7584	if (iSegReg == X86_SREG_SS)
7585	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7586	else
7587	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7588	return VINF_HM_PENDING_XCPT;
7589	}
7590	}
7591	else
7592	{
7593	/* Check limits if it's an expand-down data segment.
7594	Note! The upper boundary is defined by the B bit, not the G bit! */
7595	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
7596	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
7597	{
7598	Log4Func(("Expand-down data segment limit exceeded. "
7599	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7600	GCPtrLast32, pSel->u32Limit));
7601	if (iSegReg == X86_SREG_SS)
7602	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7603	else
7604	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7605	return VINF_HM_PENDING_XCPT;
7606	}
7607	}
7608	}
7609	else
7610	{
7611	/* Check permissions for the code segment. */
7612	if ( enmMemAccess == VMXMEMACCESS_WRITE
7613	\|\| ( enmMemAccess == VMXMEMACCESS_READ
7614	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
7615	{
7616	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
7617	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7618	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7619	return VINF_HM_PENDING_XCPT;
7620	}
7621
7622	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
7623	if ( GCPtrFirst32 > pSel->u32Limit
7624	\|\| GCPtrLast32 > pSel->u32Limit)
7625	{
7626	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
7627	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
7628	if (iSegReg == X86_SREG_SS)
7629	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7630	else
7631	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7632	return VINF_HM_PENDING_XCPT;
7633	}
7634	}
7635	}
7636	else
7637	{
7638	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7639	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7640	return VINF_HM_PENDING_XCPT;
7641	}
7642
7643	*pGCPtrMem = GCPtrMem;
7644	return VINF_SUCCESS;
7645	}
7646
7647
7648	/**
7649	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
7650	* guest attempting to execute a VMX instruction.
7651	*
7652	* @returns Strict VBox status code (i.e. informational status codes too).
7653	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
7654	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
7655	*
7656	* @param pVCpu The cross context virtual CPU structure.
7657	* @param uExitReason The VM-exit reason.
7658	*
7659	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
7660	* @remarks No-long-jump zone!!!
7661	*/
7662	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
7663	{
7664	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
7665	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7666
7667	/*
7668	* The physical CPU would have already checked the CPU mode/code segment.
7669	* We shall just assert here for paranoia.
7670	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7671	*/
7672	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7673	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7674	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7675
7676	if (uExitReason == VMX_EXIT_VMXON)
7677	{
7678	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7679
7680	/*
7681	* We check CR4.VMXE because it is required to be always set while in VMX operation
7682	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
7683	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7684	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7685	*/
7686	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7687	{
7688	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7689	hmR0VmxSetPendingXcptUD(pVCpu);
7690	return VINF_HM_PENDING_XCPT;
7691	}
7692	}
7693	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7694	{
7695	/*
7696	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7697	* (other than VMXON), we need to raise a #UD.
7698	*/
7699	Log4Func(("Not in VMX root mode -> #UD\n"));
7700	hmR0VmxSetPendingXcptUD(pVCpu);
7701	return VINF_HM_PENDING_XCPT;
7702	}
7703
7704	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7705	return VINF_SUCCESS;
7706	}
7707	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7708
7709
7710	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7711	{
7712	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7713
7714	/*
7715	* If VT-x marks the segment as unusable, most other bits remain undefined:
7716	* - For CS the L, D and G bits have meaning.
7717	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7718	* - For the remaining data segments no bits are defined.
7719	*
7720	* The present bit and the unusable bit has been observed to be set at the
7721	* same time (the selector was supposed to be invalid as we started executing
7722	* a V8086 interrupt in ring-0).
7723	*
7724	* What should be important for the rest of the VBox code, is that the P bit is
7725	* cleared. Some of the other VBox code recognizes the unusable bit, but
7726	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7727	* safe side here, we'll strip off P and other bits we don't care about. If
7728	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7729	*
7730	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7731	*/
7732	#ifdef VBOX_STRICT
7733	uint32_t const uAttr = pSelReg->Attr.u;
7734	#endif
7735
7736	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7737	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7738	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7739
7740	#ifdef VBOX_STRICT
7741	VMMRZCallRing3Disable(pVCpu);
7742	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7743	# ifdef DEBUG_bird
7744	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7745	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7746	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7747	# endif
7748	VMMRZCallRing3Enable(pVCpu);
7749	NOREF(uAttr);
7750	#endif
7751	RT_NOREF2(pVCpu, idxSel);
7752	}
7753
7754
7755	/**
7756	* Imports a guest segment register from the current VMCS into the guest-CPU
7757	* context.
7758	*
7759	* @returns VBox status code.
7760	* @param pVCpu The cross context virtual CPU structure.
7761	* @param iSegReg The segment register number (X86_SREG_XXX).
7762	*
7763	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7764	* do not log!
7765	*/
7766	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7767	{
7768	Assert(iSegReg < X86_SREG_COUNT);
7769
7770	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7771	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7772	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7773	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7774	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7775	#else
7776	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7777	#endif
7778	uint64_t u64Base;
7779	uint32_t u32Sel, u32Limit, u32Attr;
7780	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7781	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7782	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7783	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7784	if (RT_SUCCESS(rc))
7785	{
7786	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7787	pSelReg->Sel = u32Sel;
7788	pSelReg->ValidSel = u32Sel;
7789	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7790	pSelReg->u32Limit = u32Limit;
7791	pSelReg->u64Base = u64Base;
7792	pSelReg->Attr.u = u32Attr;
7793	if (u32Attr & X86DESCATTR_UNUSABLE)
7794	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7795	}
7796	return rc;
7797	}
7798
7799
7800	/**
7801	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7802	*
7803	* @returns VBox status code.
7804	* @param pVCpu The cross context virtual CPU structure.
7805	*
7806	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7807	* do not log!
7808	*/
7809	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7810	{
7811	uint64_t u64Base;
7812	uint32_t u32Sel, u32Limit, u32Attr;
7813	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7814	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7815	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7816	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7817	if (RT_SUCCESS(rc))
7818	{
7819	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7820	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7821	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7822	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7823	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7824	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7825	if (u32Attr & X86DESCATTR_UNUSABLE)
7826	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7827	}
7828	return rc;
7829	}
7830
7831
7832	/**
7833	* Imports the guest TR from the current VMCS into the guest-CPU context.
7834	*
7835	* @returns VBox status code.
7836	* @param pVCpu The cross context virtual CPU structure.
7837	*
7838	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7839	* do not log!
7840	*/
7841	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7842	{
7843	uint32_t u32Sel, u32Limit, u32Attr;
7844	uint64_t u64Base;
7845	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7846	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7847	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7848	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7849	AssertRCReturn(rc, rc);
7850
7851	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7852	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7853	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7854	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7855	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7856	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7857	/* TR is the only selector that can never be unusable. */
7858	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7859	return VINF_SUCCESS;
7860	}
7861
7862
7863	/**
7864	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7865	*
7866	* @returns VBox status code.
7867	* @param pVCpu The cross context virtual CPU structure.
7868	*
7869	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7870	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7871	* instead!!!
7872	*/
7873	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7874	{
7875	uint64_t u64Val;
7876	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7877	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7878	{
7879	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7880	if (RT_SUCCESS(rc))
7881	{
7882	pCtx->rip = u64Val;
7883	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7884	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7885	}
7886	return rc;
7887	}
7888	return VINF_SUCCESS;
7889	}
7890
7891
7892	/**
7893	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7894	*
7895	* @returns VBox status code.
7896	* @param pVCpu The cross context virtual CPU structure.
7897	* @param pVmcsInfo The VMCS info. object.
7898	*
7899	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7900	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7901	* instead!!!
7902	*/
7903	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7904	{
7905	uint32_t u32Val;
7906	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7907	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7908	{
7909	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7910	if (RT_SUCCESS(rc))
7911	{
7912	pCtx->eflags.u32 = u32Val;
7913
7914	/* Restore eflags for real-on-v86-mode hack. */
7915	if (pVmcsInfo->RealMode.fRealOnV86Active)
7916	{
7917	pCtx->eflags.Bits.u1VM = 0;
7918	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7919	}
7920	}
7921	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7922	return rc;
7923	}
7924	return VINF_SUCCESS;
7925	}
7926
7927
7928	/**
7929	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7930	* context.
7931	*
7932	* @returns VBox status code.
7933	* @param pVCpu The cross context virtual CPU structure.
7934	* @param pVmcsInfo The VMCS info. object.
7935	*
7936	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7937	* do not log!
7938	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7939	* instead!!!
7940	*/
7941	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7942	{
7943	uint32_t u32Val;
7944	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7945	if (RT_SUCCESS(rc))
7946	{
7947	if (!u32Val)
7948	{
7949	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7950	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7951
7952	CPUMSetGuestNmiBlocking(pVCpu, false);
7953	}
7954	else
7955	{
7956	/*
7957	* We must import RIP here to set our EM interrupt-inhibited state.
7958	* We also import RFLAGS as our code that evaluates pending interrupts
7959	* before VM-entry requires it.
7960	*/
7961	rc = hmR0VmxImportGuestRip(pVCpu);
7962	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7963	if (RT_SUCCESS(rc))
7964	{
7965	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7966	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7967	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7968	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7969
7970	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7971	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7972	}
7973	}
7974	}
7975	return rc;
7976	}
7977
7978
7979	/**
7980	* Worker for VMXR0ImportStateOnDemand.
7981	*
7982	* @returns VBox status code.
7983	* @param pVCpu The cross context virtual CPU structure.
7984	* @param pVmcsInfo The VMCS info. object.
7985	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7986	*/
7987	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7988	{
7989	#define VMXLOCAL_BREAK_RC(a_rc) \
7990	if (RT_SUCCESS(a_rc)) \
7991	{ } \
7992	else \
7993	break
7994
7995	int rc = VINF_SUCCESS;
7996	PVM pVM = pVCpu->CTX_SUFF(pVM);
7997	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7998	uint64_t u64Val;
7999	uint32_t u32Val;
8000
8001	/*
8002	* Note! This is hack to workaround a mysterious BSOD observed with release builds
8003	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8004	* neither are other host platforms.
8005	*
8006	* Committing this temporarily as it prevents BSOD.
8007	*
8008	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8009	*/
8010	#ifdef RT_OS_WINDOWS
8011	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
8012	return VERR_HM_IPE_1;
8013	#endif
8014
8015	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8016
8017	/*
8018	* We disable interrupts to make the updating of the state and in particular
8019	* the fExtrn modification atomic wrt to preemption hooks.
8020	*/
8021	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8022
8023	fWhat &= pCtx->fExtrn;
8024	if (fWhat)
8025	{
8026	do
8027	{
8028	if (fWhat & CPUMCTX_EXTRN_RIP)
8029	{
8030	rc = hmR0VmxImportGuestRip(pVCpu);
8031	VMXLOCAL_BREAK_RC(rc);
8032	}
8033
8034	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8035	{
8036	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8037	VMXLOCAL_BREAK_RC(rc);
8038	}
8039
8040	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8041	{
8042	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8043	VMXLOCAL_BREAK_RC(rc);
8044	}
8045
8046	if (fWhat & CPUMCTX_EXTRN_RSP)
8047	{
8048	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8049	VMXLOCAL_BREAK_RC(rc);
8050	pCtx->rsp = u64Val;
8051	}
8052
8053	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8054	{
8055	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8056	if (fWhat & CPUMCTX_EXTRN_CS)
8057	{
8058	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8059	rc \|= hmR0VmxImportGuestRip(pVCpu);
8060	if (fRealOnV86Active)
8061	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8062	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8063	}
8064	if (fWhat & CPUMCTX_EXTRN_SS)
8065	{
8066	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8067	if (fRealOnV86Active)
8068	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8069	}
8070	if (fWhat & CPUMCTX_EXTRN_DS)
8071	{
8072	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8073	if (fRealOnV86Active)
8074	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8075	}
8076	if (fWhat & CPUMCTX_EXTRN_ES)
8077	{
8078	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8079	if (fRealOnV86Active)
8080	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8081	}
8082	if (fWhat & CPUMCTX_EXTRN_FS)
8083	{
8084	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8085	if (fRealOnV86Active)
8086	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8087	}
8088	if (fWhat & CPUMCTX_EXTRN_GS)
8089	{
8090	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8091	if (fRealOnV86Active)
8092	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8093	}
8094	VMXLOCAL_BREAK_RC(rc);
8095	}
8096
8097	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8098	{
8099	if (fWhat & CPUMCTX_EXTRN_LDTR)
8100	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
8101
8102	if (fWhat & CPUMCTX_EXTRN_GDTR)
8103	{
8104	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8105	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8106	pCtx->gdtr.pGdt = u64Val;
8107	pCtx->gdtr.cbGdt = u32Val;
8108	}
8109
8110	/* Guest IDTR. */
8111	if (fWhat & CPUMCTX_EXTRN_IDTR)
8112	{
8113	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8114	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8115	pCtx->idtr.pIdt = u64Val;
8116	pCtx->idtr.cbIdt = u32Val;
8117	}
8118
8119	/* Guest TR. */
8120	if (fWhat & CPUMCTX_EXTRN_TR)
8121	{
8122	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8123	don't need to import that one. */
8124	if (!pVmcsInfo->RealMode.fRealOnV86Active)
8125	rc \|= hmR0VmxImportGuestTr(pVCpu);
8126	}
8127	VMXLOCAL_BREAK_RC(rc);
8128	}
8129
8130	if (fWhat & CPUMCTX_EXTRN_DR7)
8131	{
8132	if (!pVCpu->hm.s.fUsingHyperDR7)
8133	{
8134	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8135	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8136	VMXLOCAL_BREAK_RC(rc);
8137	pCtx->dr[7] = u32Val;
8138	}
8139	}
8140
8141	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8142	{
8143	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8144	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8145	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8146	pCtx->SysEnter.cs = u32Val;
8147	VMXLOCAL_BREAK_RC(rc);
8148	}
8149
8150	#if HC_ARCH_BITS == 64
8151	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8152	{
8153	if ( pVM->hm.s.fAllow64BitGuests
8154	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8155	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8156	}
8157
8158	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8159	{
8160	if ( pVM->hm.s.fAllow64BitGuests
8161	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8162	{
8163	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8164	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8165	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8166	}
8167	}
8168	#endif
8169
8170	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
8171	#if HC_ARCH_BITS == 32
8172	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
8173	#endif
8174	)
8175	{
8176	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8177	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8178	Assert(pMsrs);
8179	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8180	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
8181	for (uint32_t i = 0; i < cMsrs; i++)
8182	{
8183	uint32_t const idMsr = pMsrs[i].u32Msr;
8184	switch (idMsr)
8185	{
8186	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8187	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8188	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8189	#if HC_ARCH_BITS == 32
8190	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8191	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8192	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8193	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8194	#endif
8195	default:
8196	{
8197	pCtx->fExtrn = 0;
8198	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8199	ASMSetFlags(fEFlags);
8200	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8201	return VERR_HM_UNEXPECTED_LD_ST_MSR;
8202	}
8203	}
8204	}
8205	}
8206
8207	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8208	{
8209	uint64_t u64Shadow;
8210	if (fWhat & CPUMCTX_EXTRN_CR0)
8211	{
8212	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8213	* remove when we drop 32-bit host w/ 64-bit host support, see
8214	* @bugref{9180#c39}. */
8215	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8216	#if HC_ARCH_BITS == 32
8217	uint32_t u32Shadow;
8218	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8219	u64Shadow = u32Shadow;
8220	#else
8221	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8222	#endif
8223	VMXLOCAL_BREAK_RC(rc);
8224	u64Val = u32Val;
8225	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8226	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
8227	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8228	/*
8229	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
8230	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8231	*/
8232	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8233	{
8234	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8235	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8236	}
8237	#endif
8238	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8239	CPUMSetGuestCR0(pVCpu, u64Val);
8240	VMMRZCallRing3Enable(pVCpu);
8241	}
8242
8243	if (fWhat & CPUMCTX_EXTRN_CR4)
8244	{
8245	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8246	* remove when we drop 32-bit host w/ 64-bit host support, see
8247	* @bugref{9180#c39}. */
8248	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8249	#if HC_ARCH_BITS == 32
8250	uint32_t u32Shadow;
8251	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8252	u64Shadow = u32Shadow;
8253	#else
8254	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8255	#endif
8256	VMXLOCAL_BREAK_RC(rc);
8257	u64Val = u32Val;
8258	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8259	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
8260	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8261	/*
8262	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
8263	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8264	*/
8265	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8266	{
8267	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8268	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8269	}
8270	#endif
8271	pCtx->cr4 = u64Val;
8272	}
8273
8274	if (fWhat & CPUMCTX_EXTRN_CR3)
8275	{
8276	/* CR0.PG bit changes are always intercepted, so it's up to date. */
8277	if ( pVM->hm.s.vmx.fUnrestrictedGuest
8278	\|\| ( pVM->hm.s.fNestedPaging
8279	&& CPUMIsGuestPagingEnabledEx(pCtx)))
8280	{
8281	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8282	VMXLOCAL_BREAK_RC(rc);
8283	if (pCtx->cr3 != u64Val)
8284	{
8285	pCtx->cr3 = u64Val;
8286	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8287	}
8288
8289	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8290	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8291	if (CPUMIsGuestInPAEModeEx(pCtx))
8292	{
8293	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8294	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8295	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8296	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8297	VMXLOCAL_BREAK_RC(rc);
8298	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8299	}
8300	}
8301	}
8302
8303	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8304	# if 0
8305	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8306	* VM-exit handlers as parameters. We would handle it differently when using
8307	* the fast path. */
8308	/*
8309	* The hardware virtualization state currently consists of VMCS fields that may be
8310	* modified by execution of the nested-guest (that are not part of the general
8311	* guest state) and is visible to guest software. Hence, it is technically part of
8312	* the guest-CPU state when executing a nested-guest.
8313	*/
8314	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
8315	&& CPUMIsGuestInVmxNonRootMode(pCtx))
8316	{
8317	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8318	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8319	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8320	VMXLOCAL_BREAK_RC(rc);
8321
8322	/*
8323	* VM-entry can fail due to invalid-guest state, machine-check events and
8324	* MSR loading failures. Other than VM-exit reason and Exit qualification
8325	* all other VMCS fields are left unmodified on VM-entry failure.
8326	*
8327	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8328	*/
8329	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8330	if (!fEntryFailed)
8331	{
8332	/*
8333	* Some notes on VMCS fields that may need importing when the fast path
8334	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8335	*
8336	* Requires fixing up when using hardware-assisted VMX:
8337	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8338	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
8339	* - IDT-vectoring info: Think about this.
8340	* - IDT-vectoring error code: Think about this.
8341	*
8342	* Emulated:
8343	* - Guest-interruptiblity state: Derived from FFs and RIP.
8344	* - Guest pending debug exceptions: Derived from DR6.
8345	* - Guest activity state: Emulated from EM state.
8346	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8347	* - Entry-interrupt info: Emulated, cleared to 0.
8348	*/
8349	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8350	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8351	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8352	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8353	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8354	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8355	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8356	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8357	/** @todo NSTVMX: Save and adjust preemption timer value. */
8358	}
8359
8360	VMXLOCAL_BREAK_RC(rc);
8361	}
8362	# endif
8363	#endif
8364	}
8365	} while (0);
8366
8367	if (RT_SUCCESS(rc))
8368	{
8369	/* Update fExtrn. */
8370	pCtx->fExtrn &= ~fWhat;
8371
8372	/* If everything has been imported, clear the HM keeper bit. */
8373	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8374	{
8375	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8376	Assert(!pCtx->fExtrn);
8377	}
8378	}
8379	}
8380	else
8381	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8382
8383	ASMSetFlags(fEFlags);
8384
8385	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8386
8387	if (RT_SUCCESS(rc))
8388	{ /* likely */ }
8389	else
8390	return rc;
8391
8392	/*
8393	* Honor any pending CR3 updates.
8394	*
8395	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8396	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8397	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8398	*
8399	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8400	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8401	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8402	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8403	*
8404	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8405	*/
8406	if (VMMRZCallRing3IsEnabled(pVCpu))
8407	{
8408	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8409	{
8410	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8411	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8412	}
8413
8414	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8415	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8416
8417	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8418	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8419	}
8420
8421	return VINF_SUCCESS;
8422	#undef VMXLOCAL_BREAK_RC
8423	}
8424
8425
8426	/**
8427	* Saves the guest state from the VMCS into the guest-CPU context.
8428	*
8429	* @returns VBox status code.
8430	* @param pVCpu The cross context virtual CPU structure.
8431	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8432	*/
8433	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8434	{
8435	AssertPtr(pVCpu);
8436	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8437	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8438	}
8439
8440
8441	/**
8442	* Check per-VM and per-VCPU force flag actions that require us to go back to
8443	* ring-3 for one reason or another.
8444	*
8445	* @returns Strict VBox status code (i.e. informational status codes too)
8446	* @retval VINF_SUCCESS if we don't have any actions that require going back to
8447	* ring-3.
8448	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8449	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8450	* interrupts)
8451	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8452	* all EMTs to be in ring-3.
8453	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8454	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8455	* to the EM loop.
8456	*
8457	* @param pVCpu The cross context virtual CPU structure.
8458	* @param fStepping Whether we are single-stepping the guest using the
8459	* hypervisor debugger.
8460	*/
8461	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8462	{
8463	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8464
8465	/*
8466	* Update pending interrupts into the APIC's IRR.
8467	*/
8468	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8469	APICUpdatePendingInterrupts(pVCpu);
8470
8471	/*
8472	* Anything pending? Should be more likely than not if we're doing a good job.
8473	*/
8474	PVM pVM = pVCpu->CTX_SUFF(pVM);
8475	if ( !fStepping
8476	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8477	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8478	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8479	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8480	return VINF_SUCCESS;
8481
8482	/* Pending PGM C3 sync. */
8483	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8484	{
8485	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8486	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8487	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8488	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8489	if (rcStrict2 != VINF_SUCCESS)
8490	{
8491	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8492	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8493	return rcStrict2;
8494	}
8495	}
8496
8497	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8498	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8499	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8500	{
8501	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8502	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8503	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8504	return rc2;
8505	}
8506
8507	/* Pending VM request packets, such as hardware interrupts. */
8508	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8509	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8510	{
8511	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8512	return VINF_EM_PENDING_REQUEST;
8513	}
8514
8515	/* Pending PGM pool flushes. */
8516	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8517	{
8518	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8519	return VINF_PGM_POOL_FLUSH_PENDING;
8520	}
8521
8522	/* Pending DMA requests. */
8523	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8524	{
8525	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8526	return VINF_EM_RAW_TO_R3;
8527	}
8528
8529	return VINF_SUCCESS;
8530	}
8531
8532
8533	/**
8534	* Converts any TRPM trap into a pending HM event. This is typically used when
8535	* entering from ring-3 (not longjmp returns).
8536	*
8537	* @param pVCpu The cross context virtual CPU structure.
8538	*/
8539	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8540	{
8541	Assert(TRPMHasTrap(pVCpu));
8542	Assert(!pVCpu->hm.s.Event.fPending);
8543
8544	uint8_t uVector;
8545	TRPMEVENT enmTrpmEvent;
8546	RTGCUINT uErrCode;
8547	RTGCUINTPTR GCPtrFaultAddress;
8548	uint8_t cbInstr;
8549
8550	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8551	AssertRC(rc);
8552
8553	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
8554	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
8555	if (enmTrpmEvent == TRPM_TRAP)
8556	{
8557	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
8558	* generated using INT1 (ICEBP). */
8559	switch (uVector)
8560	{
8561	case X86_XCPT_NMI:
8562	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8563	break;
8564
8565	case X86_XCPT_BP:
8566	case X86_XCPT_OF:
8567	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8568	break;
8569
8570	case X86_XCPT_PF:
8571	case X86_XCPT_DF:
8572	case X86_XCPT_TS:
8573	case X86_XCPT_NP:
8574	case X86_XCPT_SS:
8575	case X86_XCPT_GP:
8576	case X86_XCPT_AC:
8577	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
8578	RT_FALL_THRU();
8579	default:
8580	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8581	break;
8582	}
8583	}
8584	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
8585	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8586	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
8587	{
8588	switch (uVector)
8589	{
8590	case X86_XCPT_BP:
8591	case X86_XCPT_OF:
8592	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8593	break;
8594
8595	default:
8596	Assert(uVector == X86_XCPT_DB);
8597	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8598	break;
8599	}
8600	}
8601	else
8602	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
8603
8604	rc = TRPMResetTrap(pVCpu);
8605	AssertRC(rc);
8606	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8607	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8608
8609	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8610	}
8611
8612
8613	/**
8614	* Converts the pending HM event into a TRPM trap.
8615	*
8616	* @param pVCpu The cross context virtual CPU structure.
8617	*/
8618	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8619	{
8620	Assert(pVCpu->hm.s.Event.fPending);
8621
8622	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8623	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8624	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8625	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8626
8627	/* If a trap was already pending, we did something wrong! */
8628	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8629
8630	/** @todo Use HMVmxEventToTrpmEventType() later. */
8631	TRPMEVENT enmTrapType;
8632	switch (uVectorType)
8633	{
8634	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8635	enmTrapType = TRPM_HARDWARE_INT;
8636	break;
8637
8638	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8639	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8640	enmTrapType = TRPM_TRAP;
8641	break;
8642
8643	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8644	Assert(uVector == X86_XCPT_DB);
8645	enmTrapType = TRPM_SOFTWARE_INT;
8646	break;
8647
8648	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8649	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8650	enmTrapType = TRPM_SOFTWARE_INT;
8651	break;
8652
8653	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8654	enmTrapType = TRPM_SOFTWARE_INT;
8655	break;
8656
8657	default:
8658	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8659	enmTrapType = TRPM_32BIT_HACK;
8660	break;
8661	}
8662
8663	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8664
8665	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8666	AssertRC(rc);
8667
8668	if (fErrorCodeValid)
8669	TRPMSetErrorCode(pVCpu, uErrorCode);
8670
8671	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8672	&& uVector == X86_XCPT_PF)
8673	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8674	else if (enmTrapType == TRPM_SOFTWARE_INT)
8675	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8676
8677	/* We're now done converting the pending event. */
8678	pVCpu->hm.s.Event.fPending = false;
8679	}
8680
8681
8682	/**
8683	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8684	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8685	*
8686	* @param pVCpu The cross context virtual CPU structure.
8687	* @param pVmcsInfo The VMCS info. object.
8688	*/
8689	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8690	{
8691	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8692	{
8693	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8694	{
8695	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8696	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8697	AssertRC(rc);
8698	}
8699	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8700	}
8701
8702
8703	/**
8704	* Clears the interrupt-window exiting control in the VMCS.
8705	*
8706	* @param pVmcsInfo The VMCS info. object.
8707	*/
8708	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8709	{
8710	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8711	{
8712	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8713	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8714	}
8715	return VINF_SUCCESS;
8716	}
8717
8718
8719	/**
8720	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8721	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8722	*
8723	* @param pVCpu The cross context virtual CPU structure.
8724	* @param pVmcsInfo The VMCS info. object.
8725	*/
8726	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8727	{
8728	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8729	{
8730	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8731	{
8732	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8733	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8734	AssertRC(rc);
8735	Log4Func(("Setup NMI-window exiting\n"));
8736	}
8737	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8738	}
8739
8740
8741	/**
8742	* Clears the NMI-window exiting control in the VMCS.
8743	*
8744	* @param pVmcsInfo The VMCS info. object.
8745	*/
8746	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8747	{
8748	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8749	{
8750	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8751	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8752	}
8753	return VINF_SUCCESS;
8754	}
8755
8756
8757	/**
8758	* Does the necessary state syncing before returning to ring-3 for any reason
8759	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8760	*
8761	* @returns VBox status code.
8762	* @param pVCpu The cross context virtual CPU structure.
8763	* @param fImportState Whether to import the guest state from the VMCS back
8764	* to the guest-CPU context.
8765	*
8766	* @remarks No-long-jmp zone!!!
8767	*/
8768	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8769	{
8770	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8771	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8772
8773	RTCPUID idCpu = RTMpCpuId();
8774	Log4Func(("HostCpuId=%u\n", idCpu));
8775
8776	/*
8777	* !!! IMPORTANT !!!
8778	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8779	*/
8780
8781	/* Save the guest state if necessary. */
8782	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8783	if (fImportState)
8784	{
8785	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8786	AssertRCReturn(rc, rc);
8787	}
8788
8789	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8790	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8791	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8792
8793	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8794	#ifdef VBOX_STRICT
8795	if (CPUMIsHyperDebugStateActive(pVCpu))
8796	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8797	#endif
8798	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8799	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8800	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8801
8802	#if HC_ARCH_BITS == 64
8803	/* Restore host-state bits that VT-x only restores partially. */
8804	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8805	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8806	{
8807	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8808	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8809	}
8810	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8811	#endif
8812
8813	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8814	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8815	{
8816	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8817	if (!fImportState)
8818	{
8819	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8820	AssertRCReturn(rc, rc);
8821	}
8822	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8823	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8824	}
8825	else
8826	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8827
8828	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8829	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8830
8831	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8832	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8833	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8834	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8835	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8836	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8837	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8838	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8839	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8840
8841	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8842
8843	/** @todo This partially defeats the purpose of having preemption hooks.
8844	* The problem is, deregistering the hooks should be moved to a place that
8845	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8846	* context.
8847	*/
8848	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8849	AssertRCReturn(rc, rc);
8850
8851	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8852	NOREF(idCpu);
8853	return VINF_SUCCESS;
8854	}
8855
8856
8857	/**
8858	* Leaves the VT-x session.
8859	*
8860	* @returns VBox status code.
8861	* @param pVCpu The cross context virtual CPU structure.
8862	*
8863	* @remarks No-long-jmp zone!!!
8864	*/
8865	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8866	{
8867	HM_DISABLE_PREEMPT(pVCpu);
8868	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8869	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8870	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8871
8872	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8873	and done this from the VMXR0ThreadCtxCallback(). */
8874	if (!pVCpu->hm.s.fLeaveDone)
8875	{
8876	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8877	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8878	pVCpu->hm.s.fLeaveDone = true;
8879	}
8880	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8881
8882	/*
8883	* !!! IMPORTANT !!!
8884	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8885	*/
8886
8887	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8888	/** @todo Deregistering here means we need to VMCLEAR always
8889	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8890	* for calling VMMR0ThreadCtxHookDisable here! */
8891	VMMR0ThreadCtxHookDisable(pVCpu);
8892
8893	/* Leave HM context. This takes care of local init (term). */
8894	int rc = HMR0LeaveCpu(pVCpu);
8895
8896	HM_RESTORE_PREEMPT();
8897	return rc;
8898	}
8899
8900
8901	/**
8902	* Does the necessary state syncing before doing a longjmp to ring-3.
8903	*
8904	* @returns VBox status code.
8905	* @param pVCpu The cross context virtual CPU structure.
8906	*
8907	* @remarks No-long-jmp zone!!!
8908	*/
8909	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8910	{
8911	return hmR0VmxLeaveSession(pVCpu);
8912	}
8913
8914
8915	/**
8916	* Take necessary actions before going back to ring-3.
8917	*
8918	* An action requires us to go back to ring-3. This function does the necessary
8919	* steps before we can safely return to ring-3. This is not the same as longjmps
8920	* to ring-3, this is voluntary and prepares the guest so it may continue
8921	* executing outside HM (recompiler/IEM).
8922	*
8923	* @returns VBox status code.
8924	* @param pVCpu The cross context virtual CPU structure.
8925	* @param rcExit The reason for exiting to ring-3. Can be
8926	* VINF_VMM_UNKNOWN_RING3_CALL.
8927	*/
8928	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8929	{
8930	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8931
8932	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8933	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8934	{
8935	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8936	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8937	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8938	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8939	}
8940
8941	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8942	VMMRZCallRing3Disable(pVCpu);
8943	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8944
8945	/*
8946	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8947	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8948	*
8949	* This is because execution may continue from ring-3 and we would need to inject
8950	* the event from there (hence place it back in TRPM).
8951	*/
8952	if (pVCpu->hm.s.Event.fPending)
8953	{
8954	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8955	Assert(!pVCpu->hm.s.Event.fPending);
8956
8957	/* Clear the events from the VMCS. */
8958	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8959	AssertRCReturn(rc, rc);
8960	}
8961	#ifdef VBOX_STRICT
8962	else
8963	{
8964	/*
8965	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8966	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8967	* occasionally, see @bugref{9180#c42}.
8968	*
8969	* However, if the VM-entry failed, any VM entry-interruption info. field would
8970	* be left unmodified as the event would not have been injected to the guest. In
8971	* such cases, don't assert, we're not going to continue guest execution anyway.
8972	*/
8973	uint32_t uExitReason;
8974	uint32_t uEntryIntInfo;
8975	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8976	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8977	AssertRC(rc);
8978	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8979	}
8980	#endif
8981
8982	/*
8983	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8984	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8985	* (e.g. TPR below threshold).
8986	*/
8987	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8988	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8989	AssertRCReturn(rc, rc);
8990
8991	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8992	and if we're injecting an event we should have a TRPM trap pending. */
8993	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8994	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8995	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8996	#endif
8997
8998	/* Save guest state and restore host state bits. */
8999	rc = hmR0VmxLeaveSession(pVCpu);
9000	AssertRCReturn(rc, rc);
9001	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
9002
9003	/* Thread-context hooks are unregistered at this point!!! */
9004
9005	/* Sync recompiler state. */
9006	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
9007	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
9008	\| CPUM_CHANGED_LDTR
9009	\| CPUM_CHANGED_GDTR
9010	\| CPUM_CHANGED_IDTR
9011	\| CPUM_CHANGED_TR
9012	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
9013	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
9014	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
9015	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
9016
9017	Assert(!pVCpu->hm.s.fClearTrapFlag);
9018
9019	/* Update the exit-to-ring 3 reason. */
9020	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
9021
9022	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
9023	if ( rcExit != VINF_EM_RAW_INTERRUPT
9024	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9025	{
9026	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
9027	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9028	}
9029
9030	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
9031
9032	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
9033	VMMRZCallRing3RemoveNotification(pVCpu);
9034	VMMRZCallRing3Enable(pVCpu);
9035
9036	return rc;
9037	}
9038
9039
9040	/**
9041	* VMMRZCallRing3() callback wrapper which saves the guest state before we
9042	* longjump to ring-3 and possibly get preempted.
9043	*
9044	* @returns VBox status code.
9045	* @param pVCpu The cross context virtual CPU structure.
9046	* @param enmOperation The operation causing the ring-3 longjump.
9047	* @param pvUser User argument, currently unused, NULL.
9048	*/
9049	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9050	{
9051	RT_NOREF(pvUser);
9052	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9053	{
9054	/*
9055	* !!! IMPORTANT !!!
9056	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9057	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9058	*/
9059	VMMRZCallRing3RemoveNotification(pVCpu);
9060	VMMRZCallRing3Disable(pVCpu);
9061	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9062	RTThreadPreemptDisable(&PreemptState);
9063
9064	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9065	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9066	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9067	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9068
9069	#if HC_ARCH_BITS == 64
9070	/* Restore host-state bits that VT-x only restores partially. */
9071	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9072	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9073	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9074	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9075	#endif
9076
9077	/* Restore the lazy host MSRs as we're leaving VT-x context. */
9078	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9079	hmR0VmxLazyRestoreHostMsrs(pVCpu);
9080
9081	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9082	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9083	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9084
9085	/* Clear the current VMCS data back to memory. */
9086	hmR0VmxClearVmcs(pVmcsInfo);
9087
9088	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9089	VMMR0ThreadCtxHookDisable(pVCpu);
9090	HMR0LeaveCpu(pVCpu);
9091	RTThreadPreemptRestore(&PreemptState);
9092	return VINF_SUCCESS;
9093	}
9094
9095	Assert(pVCpu);
9096	Assert(pvUser);
9097	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9098	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9099
9100	VMMRZCallRing3Disable(pVCpu);
9101	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9102
9103	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9104
9105	int rc = hmR0VmxLongJmpToRing3(pVCpu);
9106	AssertRCReturn(rc, rc);
9107
9108	VMMRZCallRing3Enable(pVCpu);
9109	return VINF_SUCCESS;
9110	}
9111
9112
9113	/**
9114	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9115	* stack.
9116	*
9117	* @returns Strict VBox status code (i.e. informational status codes too).
9118	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9119	* @param pVCpu The cross context virtual CPU structure.
9120	* @param uValue The value to push to the guest stack.
9121	*/
9122	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9123	{
9124	/*
9125	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9126	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9127	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9128	*/
9129	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9130	if (pCtx->sp == 1)
9131	return VINF_EM_RESET;
9132	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9133	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9134	AssertRC(rc);
9135	return rc;
9136	}
9137
9138
9139	/**
9140	* Injects an event into the guest upon VM-entry by updating the relevant fields
9141	* in the VM-entry area in the VMCS.
9142	*
9143	* @returns Strict VBox status code (i.e. informational status codes too).
9144	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9145	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9146	*
9147	* @param pVCpu The cross context virtual CPU structure.
9148	* @param pVmxTransient The VMX-transient structure.
9149	* @param pEvent The event being injected.
9150	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
9151	* This will be updated if necessary. This cannot not
9152	* be NULL.
9153	* @param fStepping Whether we're single-stepping guest execution and
9154	* should return VINF_EM_DBG_STEPPED if the event is
9155	* injected directly (registers modified by us, not by
9156	* hardware on VM-entry).
9157	*/
9158	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9159	uint32_t *pfIntrState)
9160	{
9161	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9162	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9163	Assert(pfIntrState);
9164
9165	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9166	uint32_t u32IntInfo = pEvent->u64IntInfo;
9167	uint32_t const u32ErrCode = pEvent->u32ErrCode;
9168	uint32_t const cbInstr = pEvent->cbInstr;
9169	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9170	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9171	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9172
9173	#ifdef VBOX_STRICT
9174	/*
9175	* Validate the error-code-valid bit for hardware exceptions.
9176	* No error codes for exceptions in real-mode.
9177	*
9178	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9179	*/
9180	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9181	&& !CPUMIsGuestInRealModeEx(pCtx))
9182	{
9183	switch (uVector)
9184	{
9185	case X86_XCPT_PF:
9186	case X86_XCPT_DF:
9187	case X86_XCPT_TS:
9188	case X86_XCPT_NP:
9189	case X86_XCPT_SS:
9190	case X86_XCPT_GP:
9191	case X86_XCPT_AC:
9192	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9193	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9194	RT_FALL_THRU();
9195	default:
9196	break;
9197	}
9198	}
9199
9200	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
9201	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9202	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9203	#endif
9204
9205	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9206
9207	/*
9208	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
9209	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9210	* interrupt handler in the (real-mode) guest.
9211	*
9212	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9213	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9214	*/
9215	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9216	{
9217	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9218	{
9219	/*
9220	* For CPUs with unrestricted guest execution enabled and with the guest
9221	* in real-mode, we must not set the deliver-error-code bit.
9222	*
9223	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9224	*/
9225	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9226	}
9227	else
9228	{
9229	PVM pVM = pVCpu->CTX_SUFF(pVM);
9230	Assert(PDMVmmDevHeapIsEnabled(pVM));
9231	Assert(pVM->hm.s.vmx.pRealModeTSS);
9232	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9233
9234	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9235	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9236	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
9237	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
9238	AssertRCReturn(rc2, rc2);
9239
9240	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9241	size_t const cbIdtEntry = sizeof(X86IDTR16);
9242	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9243	{
9244	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9245	if (uVector == X86_XCPT_DF)
9246	return VINF_EM_RESET;
9247
9248	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9249	No error codes for exceptions in real-mode. */
9250	if (uVector == X86_XCPT_GP)
9251	{
9252	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9253	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9254	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9255	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9256	HMEVENT EventXcptDf;
9257	RT_ZERO(EventXcptDf);
9258	EventXcptDf.u64IntInfo = uXcptDfInfo;
9259	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9260	}
9261
9262	/*
9263	* If we're injecting an event with no valid IDT entry, inject a #GP.
9264	* No error codes for exceptions in real-mode.
9265	*
9266	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9267	*/
9268	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9269	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9270	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9271	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9272	HMEVENT EventXcptGp;
9273	RT_ZERO(EventXcptGp);
9274	EventXcptGp.u64IntInfo = uXcptGpInfo;
9275	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9276	}
9277
9278	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9279	uint16_t uGuestIp = pCtx->ip;
9280	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9281	{
9282	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9283	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
9284	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9285	}
9286	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9287	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9288
9289	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9290	X86IDTR16 IdtEntry;
9291	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9292	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9293	AssertRCReturn(rc2, rc2);
9294
9295	/* Construct the stack frame for the interrupt/exception handler. */
9296	VBOXSTRICTRC rcStrict;
9297	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9298	if (rcStrict == VINF_SUCCESS)
9299	{
9300	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9301	if (rcStrict == VINF_SUCCESS)
9302	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9303	}
9304
9305	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
9306	if (rcStrict == VINF_SUCCESS)
9307	{
9308	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
9309	pCtx->rip = IdtEntry.offSel;
9310	pCtx->cs.Sel = IdtEntry.uSel;
9311	pCtx->cs.ValidSel = IdtEntry.uSel;
9312	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9313	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9314	&& uVector == X86_XCPT_PF)
9315	pCtx->cr2 = GCPtrFault;
9316
9317	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
9318	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9319	\| HM_CHANGED_GUEST_RSP);
9320
9321	/*
9322	* If we delivered a hardware exception (other than an NMI) and if there was
9323	* block-by-STI in effect, we should clear it.
9324	*/
9325	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9326	{
9327	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9328	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9329	Log4Func(("Clearing inhibition due to STI\n"));
9330	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9331	}
9332
9333	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9334	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9335
9336	/*
9337	* The event has been truly dispatched to the guest. Mark it as no longer pending so
9338	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9339	*/
9340	pVCpu->hm.s.Event.fPending = false;
9341
9342	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9343	if (fStepping)
9344	rcStrict = VINF_EM_DBG_STEPPED;
9345	}
9346	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9347	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9348	return rcStrict;
9349	}
9350	}
9351
9352	/*
9353	* Validate.
9354	*/
9355	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9356	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9357
9358	/*
9359	* Inject the event into the VMCS.
9360	*/
9361	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9362	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9363	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9364	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9365	AssertRCReturn(rc, rc);
9366
9367	/*
9368	* Update guest CR2 if this is a page-fault.
9369	*/
9370	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9371	&& uVector == X86_XCPT_PF)
9372	pCtx->cr2 = GCPtrFault;
9373
9374	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9375	return VINF_SUCCESS;
9376	}
9377
9378
9379	/**
9380	* Evaluates the event to be delivered to the guest and sets it as the pending
9381	* event.
9382	*
9383	* @returns Strict VBox status code (i.e. informational status codes too).
9384	* @param pVCpu The cross context virtual CPU structure.
9385	* @param pVmxTransient The VMX-transient structure.
9386	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
9387	*/
9388	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9389	{
9390	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9391	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9392
9393	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9394	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9395	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9396	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9397	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9398
9399	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9400	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9401	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9402	Assert(!TRPMHasTrap(pVCpu));
9403	Assert(pfIntrState);
9404
9405	*pfIntrState = fIntrState;
9406
9407	/*
9408	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9409	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9410	*/
9411	/** @todo SMI. SMIs take priority over NMIs. */
9412	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9413	{
9414	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9415	if ( !pVCpu->hm.s.Event.fPending
9416	&& !fBlockNmi
9417	&& !fBlockSti
9418	&& !fBlockMovSS)
9419	{
9420	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9421	if ( pVmxTransient->fIsNestedGuest
9422	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9423	return IEMExecVmxVmexitXcptNmi(pVCpu);
9424	#endif
9425	hmR0VmxSetPendingXcptNmi(pVCpu);
9426	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9427	Log4Func(("Pending NMI\n"));
9428	}
9429	else
9430	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9431	}
9432	/*
9433	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9434	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9435	*/
9436	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
9437	&& !pVCpu->hm.s.fSingleInstruction)
9438	{
9439	Assert(!DBGFIsStepping(pVCpu));
9440	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9441	AssertRCReturn(rc, rc);
9442	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9443	if ( !pVCpu->hm.s.Event.fPending
9444	&& !fBlockInt
9445	&& !fBlockSti
9446	&& !fBlockMovSS)
9447	{
9448	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9449	if ( pVmxTransient->fIsNestedGuest
9450	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9451	{
9452	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
9453	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9454	return rcStrict;
9455	}
9456	#endif
9457	uint8_t u8Interrupt;
9458	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9459	if (RT_SUCCESS(rc))
9460	{
9461	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9462	if ( pVmxTransient->fIsNestedGuest
9463	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9464	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9465	{
9466	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9467	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9468	return rcStrict;
9469	}
9470	#endif
9471	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9472	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9473	}
9474	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9475	{
9476	if ( !pVmxTransient->fIsNestedGuest
9477	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9478	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9479	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9480
9481	/*
9482	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9483	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9484	* need to re-set this force-flag here.
9485	*/
9486	}
9487	else
9488	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9489	}
9490	else
9491	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9492	}
9493
9494	return VINF_SUCCESS;
9495	}
9496
9497
9498	/**
9499	* Injects any pending events into the guest if the guest is in a state to
9500	* receive them.
9501	*
9502	* @returns Strict VBox status code (i.e. informational status codes too).
9503	* @param pVCpu The cross context virtual CPU structure.
9504	* @param pVmxTransient The VMX-transient structure.
9505	* @param fIntrState The VT-x guest-interruptibility state.
9506	* @param fStepping Whether we are single-stepping the guest using the
9507	* hypervisor debugger and should return
9508	* VINF_EM_DBG_STEPPED if the event was dispatched
9509	* directly.
9510	*/
9511	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9512	{
9513	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9514	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9515
9516	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9517	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9518
9519	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9520	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9521	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9522	Assert(!TRPMHasTrap(pVCpu));
9523
9524	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9525	if (pVCpu->hm.s.Event.fPending)
9526	{
9527	/*
9528	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9529	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9530	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9531	*
9532	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9533	*/
9534	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9535	#ifdef VBOX_STRICT
9536	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9537	{
9538	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9539	Assert(!fBlockInt);
9540	Assert(!fBlockSti);
9541	Assert(!fBlockMovSS);
9542	}
9543	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9544	{
9545	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9546	Assert(!fBlockSti);
9547	Assert(!fBlockMovSS);
9548	Assert(!fBlockNmi);
9549	}
9550	#endif
9551	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9552	uIntType));
9553
9554	/*
9555	* Inject the event and get any changes to the guest-interruptibility state.
9556	*
9557	* The guest-interruptibility state may need to be updated if we inject the event
9558	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9559	*/
9560	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9561	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9562
9563	/*
9564	* If we are executing a nested-guest make sure that we should intercept subsequent
9565	* events. The one we are injecting might be part of VM-entry.
9566	*/
9567	if (pVmxTransient->fIsNestedGuest)
9568	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
9569
9570	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9571	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9572	else
9573	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9574	}
9575
9576	/*
9577	* Update the guest-interruptibility state.
9578	*
9579	* This is required for the real-on-v86 software interrupt injection case above, as well as
9580	* updates to the guest state from ring-3 or IEM/REM.
9581	*/
9582	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9583	AssertRCReturn(rc, rc);
9584
9585	/*
9586	* There's no need to clear the VM-entry interruption-information field here if we're not
9587	* injecting anything. VT-x clears the valid bit on every VM-exit.
9588	*
9589	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9590	*/
9591
9592	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9593	NOREF(fBlockMovSS); NOREF(fBlockSti);
9594	return rcStrict;
9595	}
9596
9597
9598	/**
9599	* Enters the VT-x session.
9600	*
9601	* @returns VBox status code.
9602	* @param pVCpu The cross context virtual CPU structure.
9603	*/
9604	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9605	{
9606	AssertPtr(pVCpu);
9607	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9608	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9609
9610	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9611	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9612	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9613
9614	#ifdef VBOX_STRICT
9615	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9616	RTCCUINTREG uHostCr4 = ASMGetCR4();
9617	if (!(uHostCr4 & X86_CR4_VMXE))
9618	{
9619	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9620	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9621	}
9622	#endif
9623
9624	/*
9625	* Load the appropriate VMCS as the current and active one.
9626	*/
9627	PVMXVMCSINFO pVmcsInfo;
9628	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9629	if (!fInNestedGuestMode)
9630	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9631	else
9632	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9633	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9634	if (RT_SUCCESS(rc))
9635	{
9636	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9637	pVCpu->hm.s.fLeaveDone = false;
9638	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9639
9640	/*
9641	* Do the EMT scheduled L1D flush here if needed.
9642	*/
9643	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9644	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9645	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9646	hmR0MdsClear();
9647	}
9648	return rc;
9649	}
9650
9651
9652	/**
9653	* The thread-context callback (only on platforms which support it).
9654	*
9655	* @param enmEvent The thread-context event.
9656	* @param pVCpu The cross context virtual CPU structure.
9657	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9658	* @thread EMT(pVCpu)
9659	*/
9660	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9661	{
9662	AssertPtr(pVCpu);
9663	RT_NOREF1(fGlobalInit);
9664
9665	switch (enmEvent)
9666	{
9667	case RTTHREADCTXEVENT_OUT:
9668	{
9669	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9670	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9671	VMCPU_ASSERT_EMT(pVCpu);
9672
9673	/* No longjmps (logger flushes, locks) in this fragile context. */
9674	VMMRZCallRing3Disable(pVCpu);
9675	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9676
9677	/* Restore host-state (FPU, debug etc.) */
9678	if (!pVCpu->hm.s.fLeaveDone)
9679	{
9680	/*
9681	* Do -not- import the guest-state here as we might already be in the middle of importing
9682	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9683	*/
9684	hmR0VmxLeave(pVCpu, false /* fImportState */);
9685	pVCpu->hm.s.fLeaveDone = true;
9686	}
9687
9688	/* Leave HM context, takes care of local init (term). */
9689	int rc = HMR0LeaveCpu(pVCpu);
9690	AssertRC(rc);
9691
9692	/* Restore longjmp state. */
9693	VMMRZCallRing3Enable(pVCpu);
9694	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9695	break;
9696	}
9697
9698	case RTTHREADCTXEVENT_IN:
9699	{
9700	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9701	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9702	VMCPU_ASSERT_EMT(pVCpu);
9703
9704	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9705	VMMRZCallRing3Disable(pVCpu);
9706	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9707
9708	/* Initialize the bare minimum state required for HM. This takes care of
9709	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9710	int rc = hmR0EnterCpu(pVCpu);
9711	AssertRC(rc);
9712	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9713	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9714
9715	/* Load the active VMCS as the current one. */
9716	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9717	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9718	AssertRC(rc);
9719	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9720	pVCpu->hm.s.fLeaveDone = false;
9721
9722	/* Do the EMT scheduled L1D flush if needed. */
9723	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9724	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9725
9726	/* Restore longjmp state. */
9727	VMMRZCallRing3Enable(pVCpu);
9728	break;
9729	}
9730
9731	default:
9732	break;
9733	}
9734	}
9735
9736
9737	/**
9738	* Exports the host state into the VMCS host-state area.
9739	* Sets up the VM-exit MSR-load area.
9740	*
9741	* The CPU state will be loaded from these fields on every successful VM-exit.
9742	*
9743	* @returns VBox status code.
9744	* @param pVCpu The cross context virtual CPU structure.
9745	*
9746	* @remarks No-long-jump zone!!!
9747	*/
9748	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9749	{
9750	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9751
9752	int rc = VINF_SUCCESS;
9753	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9754	{
9755	rc = hmR0VmxExportHostControlRegs();
9756	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9757
9758	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9759	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9760
9761	rc = hmR0VmxExportHostMsrs(pVCpu);
9762	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9763
9764	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9765	}
9766	return rc;
9767	}
9768
9769
9770	/**
9771	* Saves the host state in the VMCS host-state.
9772	*
9773	* @returns VBox status code.
9774	* @param pVCpu The cross context virtual CPU structure.
9775	*
9776	* @remarks No-long-jump zone!!!
9777	*/
9778	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9779	{
9780	AssertPtr(pVCpu);
9781	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9782
9783	/*
9784	* Export the host state here while entering HM context.
9785	* When thread-context hooks are used, we might get preempted and have to re-save the host
9786	* state but most of the time we won't be, so do it here before we disable interrupts.
9787	*/
9788	return hmR0VmxExportHostState(pVCpu);
9789	}
9790
9791
9792	/**
9793	* Exports the guest state into the VMCS guest-state area.
9794	*
9795	* The will typically be done before VM-entry when the guest-CPU state and the
9796	* VMCS state may potentially be out of sync.
9797	*
9798	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9799	* VM-entry controls.
9800	* Sets up the appropriate VMX non-root function to execute guest code based on
9801	* the guest CPU mode.
9802	*
9803	* @returns VBox strict status code.
9804	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9805	* without unrestricted guest execution and the VMMDev is not presently
9806	* mapped (e.g. EFI32).
9807	*
9808	* @param pVCpu The cross context virtual CPU structure.
9809	* @param pVmxTransient The VMX-transient structure.
9810	*
9811	* @remarks No-long-jump zone!!!
9812	*/
9813	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9814	{
9815	AssertPtr(pVCpu);
9816	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9817	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9818
9819	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9820
9821	/*
9822	* Determine real-on-v86 mode.
9823	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9824	*/
9825	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9826	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9827	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9828	pVmcsInfo->RealMode. fRealOnV86Active = false;
9829	else
9830	{
9831	Assert(!pVmxTransient->fIsNestedGuest);
9832	pVmcsInfo->RealMode.fRealOnV86Active = true;
9833	}
9834
9835	/*
9836	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9837	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9838	*/
9839	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9840	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9841	* be a need to evaluate this everytime since I'm pretty sure we intercept
9842	* all guest paging mode changes. */
9843	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9844	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9845
9846	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9847	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9848
9849	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9850	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9851
9852	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9853	if (rcStrict == VINF_SUCCESS)
9854	{ /* likely */ }
9855	else
9856	{
9857	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9858	return rcStrict;
9859	}
9860
9861	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9862	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9863
9864	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9865	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9866
9867	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9868	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9869
9870	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9871	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9872
9873	rc = hmR0VmxExportGuestRip(pVCpu);
9874	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9875	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9876	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9877
9878	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9879	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9880	\| HM_CHANGED_GUEST_CR2
9881	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9882	\| HM_CHANGED_GUEST_X87
9883	\| HM_CHANGED_GUEST_SSE_AVX
9884	\| HM_CHANGED_GUEST_OTHER_XSAVE
9885	\| HM_CHANGED_GUEST_XCRx
9886	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9887	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9888	\| HM_CHANGED_GUEST_TSC_AUX
9889	\| HM_CHANGED_GUEST_OTHER_MSRS
9890	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9891	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9892
9893	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9894	return rc;
9895	}
9896
9897
9898	/**
9899	* Exports the state shared between the host and guest into the VMCS.
9900	*
9901	* @param pVCpu The cross context virtual CPU structure.
9902	* @param pVmxTransient The VMX-transient structure.
9903	*
9904	* @remarks No-long-jump zone!!!
9905	*/
9906	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9907	{
9908	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9909	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9910
9911	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9912	{
9913	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9914	AssertRC(rc);
9915	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9916
9917	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9918	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9919	{
9920	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9921	AssertRC(rc);
9922	}
9923	}
9924
9925	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9926	{
9927	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9928	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9929	}
9930
9931	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9932	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9933	}
9934
9935
9936	/**
9937	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9938	*
9939	* @returns Strict VBox status code (i.e. informational status codes too).
9940	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9941	* without unrestricted guest execution and the VMMDev is not presently
9942	* mapped (e.g. EFI32).
9943	*
9944	* @param pVCpu The cross context virtual CPU structure.
9945	* @param pVmxTransient The VMX-transient structure.
9946	*
9947	* @remarks No-long-jump zone!!!
9948	*/
9949	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9950	{
9951	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9952	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9953	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9954
9955	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9956	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9957	#endif
9958
9959	/*
9960	* For many exits it's only RIP that changes and hence try to export it first
9961	* without going through a lot of change flag checks.
9962	*/
9963	VBOXSTRICTRC rcStrict;
9964	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9965	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9966	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9967	{
9968	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9969	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9970	{ /* likely */}
9971	else
9972	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9973	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9974	}
9975	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9976	{
9977	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9978	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9979	{ /* likely */}
9980	else
9981	{
9982	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9983	VBOXSTRICTRC_VAL(rcStrict)));
9984	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9985	return rcStrict;
9986	}
9987	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9988	}
9989	else
9990	rcStrict = VINF_SUCCESS;
9991
9992	#ifdef VBOX_STRICT
9993	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9994	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9995	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9996	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9997	("fCtxChanged=%#RX64\n", fCtxChanged));
9998	#endif
9999	return rcStrict;
10000	}
10001
10002
10003	/**
10004	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10005	* and update error record fields accordingly.
10006	*
10007	* @returns VMX_IGS_* error codes.
10008	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10009	* wrong with the guest state.
10010	*
10011	* @param pVCpu The cross context virtual CPU structure.
10012	* @param pVmcsInfo The VMCS info. object.
10013	*
10014	* @remarks This function assumes our cache of the VMCS controls
10015	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10016	*/
10017	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10018	{
10019	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10020	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10021	uError = (err); \
10022	break; \
10023	} else do { } while (0)
10024
10025	int rc;
10026	PVM pVM = pVCpu->CTX_SUFF(pVM);
10027	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10028	uint32_t uError = VMX_IGS_ERROR;
10029	uint32_t u32Val;
10030	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10031
10032	do
10033	{
10034	/*
10035	* CR0.
10036	*/
10037	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10038	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10039	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10040	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10041	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10042	if (fUnrestrictedGuest)
10043	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10044
10045	uint32_t u32GuestCr0;
10046	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10047	AssertRCBreak(rc);
10048	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10049	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10050	if ( !fUnrestrictedGuest
10051	&& (u32GuestCr0 & X86_CR0_PG)
10052	&& !(u32GuestCr0 & X86_CR0_PE))
10053	{
10054	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10055	}
10056
10057	/*
10058	* CR4.
10059	*/
10060	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10061	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10062	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10063
10064	uint32_t u32GuestCr4;
10065	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10066	AssertRCBreak(rc);
10067	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10068	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10069
10070	/*
10071	* IA32_DEBUGCTL MSR.
10072	*/
10073	uint64_t u64Val;
10074	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10075	AssertRCBreak(rc);
10076	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10077	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10078	{
10079	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10080	}
10081	uint64_t u64DebugCtlMsr = u64Val;
10082
10083	#ifdef VBOX_STRICT
10084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10085	AssertRCBreak(rc);
10086	Assert(u32Val == pVmcsInfo->u32EntryCtls);
10087	#endif
10088	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10089
10090	/*
10091	* RIP and RFLAGS.
10092	*/
10093	uint32_t u32Eflags;
10094	#if HC_ARCH_BITS == 64
10095	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10096	AssertRCBreak(rc);
10097	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10098	if ( !fLongModeGuest
10099	\|\| !pCtx->cs.Attr.n.u1Long)
10100	{
10101	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10102	}
10103	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10104	* must be identical if the "IA-32e mode guest" VM-entry
10105	* control is 1 and CS.L is 1. No check applies if the
10106	* CPU supports 64 linear-address bits. */
10107
10108	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10109	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10110	AssertRCBreak(rc);
10111	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10112	VMX_IGS_RFLAGS_RESERVED);
10113	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10114	u32Eflags = u64Val;
10115	#else
10116	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10117	AssertRCBreak(rc);
10118	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10119	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10120	#endif
10121
10122	if ( fLongModeGuest
10123	\|\| ( fUnrestrictedGuest
10124	&& !(u32GuestCr0 & X86_CR0_PE)))
10125	{
10126	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10127	}
10128
10129	uint32_t u32EntryInfo;
10130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10131	AssertRCBreak(rc);
10132	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10133	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10134	{
10135	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10136	}
10137
10138	/*
10139	* 64-bit checks.
10140	*/
10141	#if HC_ARCH_BITS == 64
10142	if (fLongModeGuest)
10143	{
10144	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10145	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10146	}
10147
10148	if ( !fLongModeGuest
10149	&& (u32GuestCr4 & X86_CR4_PCIDE))
10150	{
10151	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10152	}
10153
10154	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10155	* 51:32 beyond the processor's physical-address width are 0. */
10156
10157	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10158	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10159	{
10160	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10161	}
10162
10163	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10164	AssertRCBreak(rc);
10165	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10166
10167	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10168	AssertRCBreak(rc);
10169	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10170	#endif
10171
10172	/*
10173	* PERF_GLOBAL MSR.
10174	*/
10175	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10176	{
10177	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10178	AssertRCBreak(rc);
10179	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10180	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10181	}
10182
10183	/*
10184	* PAT MSR.
10185	*/
10186	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10187	{
10188	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10189	AssertRCBreak(rc);
10190	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10191	for (unsigned i = 0; i < 8; i++)
10192	{
10193	uint8_t u8Val = (u64Val & 0xff);
10194	if ( u8Val != 0 /* UC */
10195	&& u8Val != 1 /* WC */
10196	&& u8Val != 4 /* WT */
10197	&& u8Val != 5 /* WP */
10198	&& u8Val != 6 /* WB */
10199	&& u8Val != 7 /* UC- */)
10200	{
10201	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10202	}
10203	u64Val >>= 8;
10204	}
10205	}
10206
10207	/*
10208	* EFER MSR.
10209	*/
10210	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10211	{
10212	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10213	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10214	AssertRCBreak(rc);
10215	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10216	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10217	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10218	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10219	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10220	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10221	* iemVmxVmentryCheckGuestState(). */
10222	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10223	\|\| !(u32GuestCr0 & X86_CR0_PG)
10224	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10225	VMX_IGS_EFER_LMA_LME_MISMATCH);
10226	}
10227
10228	/*
10229	* Segment registers.
10230	*/
10231	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10232	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10233	if (!(u32Eflags & X86_EFL_VM))
10234	{
10235	/* CS */
10236	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10237	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10238	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10239	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10240	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10241	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10242	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10243	/* CS cannot be loaded with NULL in protected mode. */
10244	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10245	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10246	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10247	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10248	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10249	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10250	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10251	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10252	else
10253	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10254
10255	/* SS */
10256	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10257	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10258	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10259	if ( !(pCtx->cr0 & X86_CR0_PE)
10260	\|\| pCtx->cs.Attr.n.u4Type == 3)
10261	{
10262	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10263	}
10264	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10265	{
10266	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10267	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10268	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10269	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10270	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10271	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10272	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10273	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10274	}
10275
10276	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10277	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10278	{
10279	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10280	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10281	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10282	\|\| pCtx->ds.Attr.n.u4Type > 11
10283	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10284	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10285	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10286	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10287	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10288	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10289	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10290	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10291	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10292	}
10293	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10294	{
10295	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10296	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10297	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10298	\|\| pCtx->es.Attr.n.u4Type > 11
10299	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10300	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10301	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10302	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10303	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10304	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10305	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10306	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10307	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10308	}
10309	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10310	{
10311	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10312	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10313	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10314	\|\| pCtx->fs.Attr.n.u4Type > 11
10315	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10316	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10317	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10318	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10319	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10320	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10321	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10322	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10323	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10324	}
10325	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10326	{
10327	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10328	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10329	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10330	\|\| pCtx->gs.Attr.n.u4Type > 11
10331	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10332	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10333	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10334	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10335	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10336	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10337	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10338	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10339	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10340	}
10341	/* 64-bit capable CPUs. */
10342	#if HC_ARCH_BITS == 64
10343	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10344	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10345	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10346	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10347	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10348	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10349	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10350	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10351	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10352	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10353	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10354	#endif
10355	}
10356	else
10357	{
10358	/* V86 mode checks. */
10359	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10360	if (pVmcsInfo->RealMode.fRealOnV86Active)
10361	{
10362	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10363	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10364	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10365	}
10366	else
10367	{
10368	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10369	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10370	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10371	}
10372
10373	/* CS */
10374	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10375	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10376	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10377	/* SS */
10378	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10379	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10380	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10381	/* DS */
10382	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10383	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10384	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10385	/* ES */
10386	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10387	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10388	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10389	/* FS */
10390	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10391	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10392	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10393	/* GS */
10394	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10395	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10396	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10397	/* 64-bit capable CPUs. */
10398	#if HC_ARCH_BITS == 64
10399	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10400	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10401	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10402	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10403	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10404	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10405	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10406	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10407	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10408	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10409	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10410	#endif
10411	}
10412
10413	/*
10414	* TR.
10415	*/
10416	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10417	/* 64-bit capable CPUs. */
10418	#if HC_ARCH_BITS == 64
10419	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10420	#endif
10421	if (fLongModeGuest)
10422	{
10423	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10424	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10425	}
10426	else
10427	{
10428	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10429	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10430	VMX_IGS_TR_ATTR_TYPE_INVALID);
10431	}
10432	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10433	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10434	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10435	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10436	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10437	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10438	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10439	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10440
10441	/*
10442	* GDTR and IDTR.
10443	*/
10444	#if HC_ARCH_BITS == 64
10445	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10446	AssertRCBreak(rc);
10447	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10448
10449	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10450	AssertRCBreak(rc);
10451	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10452	#endif
10453
10454	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10455	AssertRCBreak(rc);
10456	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10457
10458	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10459	AssertRCBreak(rc);
10460	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10461
10462	/*
10463	* Guest Non-Register State.
10464	*/
10465	/* Activity State. */
10466	uint32_t u32ActivityState;
10467	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10468	AssertRCBreak(rc);
10469	HMVMX_CHECK_BREAK( !u32ActivityState
10470	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10471	VMX_IGS_ACTIVITY_STATE_INVALID);
10472	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10473	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10474	uint32_t u32IntrState;
10475	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10476	AssertRCBreak(rc);
10477	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10478	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10479	{
10480	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10481	}
10482
10483	/** @todo Activity state and injecting interrupts. Left as a todo since we
10484	* currently don't use activity states but ACTIVE. */
10485
10486	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10487	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10488
10489	/* Guest interruptibility-state. */
10490	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10491	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10492	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10493	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10494	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10495	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10496	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10497	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10498	{
10499	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10500	{
10501	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10502	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10503	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10504	}
10505	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10506	{
10507	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10508	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10509	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10510	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10511	}
10512	}
10513	/** @todo Assumes the processor is not in SMM. */
10514	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10515	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10516	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10517	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10518	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10519	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10520	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10521	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10522	{
10523	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10524	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10525	}
10526
10527	/* Pending debug exceptions. */
10528	#if HC_ARCH_BITS == 64
10529	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10530	AssertRCBreak(rc);
10531	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10532	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10533	u32Val = u64Val; /* For pending debug exceptions checks below. */
10534	#else
10535	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10536	AssertRCBreak(rc);
10537	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10538	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10539	#endif
10540
10541	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10542	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10543	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10544	{
10545	if ( (u32Eflags & X86_EFL_TF)
10546	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10547	{
10548	/* Bit 14 is PendingDebug.BS. */
10549	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10550	}
10551	if ( !(u32Eflags & X86_EFL_TF)
10552	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10553	{
10554	/* Bit 14 is PendingDebug.BS. */
10555	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10556	}
10557	}
10558
10559	/* VMCS link pointer. */
10560	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10561	AssertRCBreak(rc);
10562	if (u64Val != UINT64_C(0xffffffffffffffff))
10563	{
10564	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10565	/** @todo Bits beyond the processor's physical-address width MBZ. */
10566	/** @todo SMM checks. */
10567	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10568	Assert(pVmcsInfo->pvShadowVmcs);
10569	VMXVMCSREVID VmcsRevId;
10570	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10571	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10572	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10573	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10574	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10575	}
10576
10577	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10578	* not using nested paging? */
10579	if ( pVM->hm.s.fNestedPaging
10580	&& !fLongModeGuest
10581	&& CPUMIsGuestInPAEModeEx(pCtx))
10582	{
10583	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10584	AssertRCBreak(rc);
10585	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10586
10587	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10588	AssertRCBreak(rc);
10589	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10590
10591	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10592	AssertRCBreak(rc);
10593	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10594
10595	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10596	AssertRCBreak(rc);
10597	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10598	}
10599
10600	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10601	if (uError == VMX_IGS_ERROR)
10602	uError = VMX_IGS_REASON_NOT_FOUND;
10603	} while (0);
10604
10605	pVCpu->hm.s.u32HMError = uError;
10606	return uError;
10607
10608	#undef HMVMX_ERROR_BREAK
10609	#undef HMVMX_CHECK_BREAK
10610	}
10611
10612
10613	/**
10614	* Setup the APIC-access page for virtualizing APIC access.
10615	*
10616	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
10617	* this not done as part of exporting guest state, see @bugref{8721}.
10618	*
10619	* @returns VBox status code.
10620	* @param pVCpu The cross context virtual CPU structure.
10621	*/
10622	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10623	{
10624	PVM pVM = pVCpu->CTX_SUFF(pVM);
10625	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10626
10627	Assert(PDMHasApic(pVM));
10628	Assert(u64MsrApicBase);
10629
10630	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10631	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10632
10633	/* Unalias any existing mapping. */
10634	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10635	AssertRCReturn(rc, rc);
10636
10637	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10638	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10639	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10640	AssertRCReturn(rc, rc);
10641
10642	/* Update the per-VCPU cache of the APIC base MSR. */
10643	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10644	return VINF_SUCCESS;
10645	}
10646
10647
10648	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10649	/**
10650	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10651	* nested-guest using hardware-assisted VMX.
10652	*
10653	* @param pVCpu The cross context virtual CPU structure.
10654	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10655	* @param pVmcsInfoGst The guest VMCS info. object.
10656	*/
10657	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10658	{
10659	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10660	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10661	Assert(pu64MsrBitmap);
10662
10663	/*
10664	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10665	* MSR that is intercepted by the guest is also intercepted while executing the
10666	* nested-guest using hardware-assisted VMX.
10667	*
10668	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10669	* nested-guest VM-exit even if the outer guest is not intercepting some
10670	* MSRs. We cannot assume the caller has initialized the nested-guest
10671	* MSR bitmap in this case.
10672	*
10673	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10674	* each VM-entry, hence initializing it once per-VM while setting up the
10675	* nested-guest VMCS is not sufficient.
10676	*/
10677	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10678	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10679	{
10680	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10681	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10682	Assert(pu64MsrBitmapNstGst);
10683	Assert(pu64MsrBitmapGst);
10684
10685	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10686	for (uint32_t i = 0; i < cFrags; i++)
10687	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10688	}
10689	else
10690	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10691	}
10692
10693
10694	/**
10695	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10696	* hardware-assisted VMX execution of the nested-guest.
10697	*
10698	* For a guest, we don't modify these controls once we set up the VMCS and hence
10699	* this function is never called.
10700	*
10701	* For nested-guests since the guest hypervisor provides these controls on every
10702	* nested-guest VM-entry and could potentially change them everytime we need to
10703	* merge them before every nested-guest VM-entry.
10704	*
10705	* @returns VBox status code.
10706	* @param pVCpu The cross context virtual CPU structure.
10707	*/
10708	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10709	{
10710	PVM pVM = pVCpu->CTX_SUFF(pVM);
10711	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10712	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10713	Assert(pVmcsNstGst);
10714
10715	/*
10716	* Merge the controls with the requirements of the guest VMCS.
10717	*
10718	* We do not need to validate the nested-guest VMX features specified in the
10719	* nested-guest VMCS with the features supported by the physical CPU as it's
10720	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10721	*
10722	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10723	* guest are derived from the VMX features supported by the physical CPU.
10724	*/
10725
10726	/* Pin-based VM-execution controls. */
10727	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10728
10729	/* Processor-based VM-execution controls. */
10730	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10731	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10732	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10733	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10734	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10735
10736	/* Secondary processor-based VM-execution controls. */
10737	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10738	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10739	\| VMX_PROC_CTLS2_INVPCID
10740	\| VMX_PROC_CTLS2_RDTSCP
10741	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10742	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10743	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10744	\| VMX_PROC_CTLS2_VMFUNC));
10745
10746	/*
10747	* VM-entry controls:
10748	* These controls contains state that depends on the nested-guest state (primarily
10749	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10750	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10751	* properly continue executing the nested-guest if the EFER MSR changes but does not
10752	* cause a nested-guest VM-exits.
10753	*
10754	* VM-exit controls:
10755	* These controls specify the host state on return. We cannot use the controls from
10756	* the guest hypervisor state as is as it would contain the guest state rather than
10757	* the host state. Since the host state is subject to change (e.g. preemption, trips
10758	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10759	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10760	*
10761	* VM-entry MSR-load:
10762	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10763	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10764	*
10765	* VM-exit MSR-store:
10766	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10767	* context back into the VM-exit MSR-store area.
10768	*
10769	* VM-exit MSR-load areas:
10770	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10771	* we can entirely ignore what the guest hypervisor wants to load here.
10772	*/
10773
10774	/*
10775	* Exception bitmap.
10776	*
10777	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10778	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10779	* keep the code more flexible if intercepting exceptions become more dynamic in
10780	* the future we do it as part of exporting the nested-guest state.
10781	*/
10782	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10783
10784	/*
10785	* CR0/CR4 guest/host mask.
10786	*
10787	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10788	* must cause VM-exits, so we need to merge them here.
10789	*/
10790	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10791	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10792
10793	/*
10794	* Page-fault error-code mask and match.
10795	*
10796	* Although we require unrestricted guest execution (and thereby nested-paging) for
10797	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10798	* normally intercept #PFs, it might intercept them for debugging purposes.
10799	*
10800	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10801	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10802	*/
10803	uint32_t u32XcptPFMask;
10804	uint32_t u32XcptPFMatch;
10805	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10806	{
10807	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10808	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10809	}
10810	else
10811	{
10812	u32XcptPFMask = 0;
10813	u32XcptPFMatch = 0;
10814	}
10815
10816	/*
10817	* Pause-Loop exiting.
10818	*/
10819	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10820	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10821
10822	/*
10823	* I/O Bitmap.
10824	*
10825	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10826	* always intercept all I/O port accesses.
10827	*/
10828	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10829
10830	/*
10831	* APIC-access page.
10832	*
10833	* The APIC-access page address has already been initialized while setting up the
10834	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10835	* should not be on any consequence to the host or to the guest for that matter, but
10836	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10837	* emulation to keep it simple.
10838	*/
10839
10840	/*
10841	* Virtual-APIC page and TPR threshold.
10842	*
10843	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10844	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10845	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10846	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10847	*/
10848	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10849	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10850	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10851	{
10852	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10853
10854	/*
10855	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10856	* we would fail to obtain a valid host-physical address for its guest-physical
10857	* address.
10858	*
10859	* We currently do not support this scenario. Maybe in the future if there is a
10860	* pressing need we can explore making this particular set of conditions work.
10861	* Right now we just cause a VM-entry failure.
10862	*
10863	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10864	* so should not really failure at the moment.
10865	*/
10866	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10867	}
10868	else
10869	{
10870	/*
10871	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10872	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10873	* be taken care of by EPT/shadow paging.
10874	*/
10875	if (pVM->hm.s.fAllow64BitGuests)
10876	{
10877	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10878	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10879	}
10880	}
10881
10882	/*
10883	* Validate basic assumptions.
10884	*/
10885	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10886	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10887	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10888	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10889
10890	/*
10891	* Commit it to the nested-guest VMCS.
10892	*/
10893	int rc = VINF_SUCCESS;
10894	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10895	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10896	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10897	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10898	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10899	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10900	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10901	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10902	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10903	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10904	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10905	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10906	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10907	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10908	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10909	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10910	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10911	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10912	{
10913	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10914	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10915	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10916	}
10917	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10918	{
10919	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10920	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10921	}
10922	AssertRCReturn(rc, rc);
10923
10924	/*
10925	* Update the nested-guest VMCS cache.
10926	*/
10927	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10928	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10929	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10930	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10931	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10932	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10933	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10934	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10935	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10936
10937	/*
10938	* MSR bitmap.
10939	*
10940	* The MSR bitmap address has already been initialized while setting up the
10941	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10942	*/
10943	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10944	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10945
10946	return VINF_SUCCESS;
10947	}
10948	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10949
10950
10951	/**
10952	* Does the preparations before executing guest code in VT-x.
10953	*
10954	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10955	* recompiler/IEM. We must be cautious what we do here regarding committing
10956	* guest-state information into the VMCS assuming we assuredly execute the
10957	* guest in VT-x mode.
10958	*
10959	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10960	* the common-state (TRPM/forceflags), we must undo those changes so that the
10961	* recompiler/IEM can (and should) use them when it resumes guest execution.
10962	* Otherwise such operations must be done when we can no longer exit to ring-3.
10963	*
10964	* @returns Strict VBox status code (i.e. informational status codes too).
10965	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10966	* have been disabled.
10967	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10968	* pending events).
10969	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10970	* double-fault into the guest.
10971	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10972	* dispatched directly.
10973	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10974	*
10975	* @param pVCpu The cross context virtual CPU structure.
10976	* @param pVmxTransient The VMX-transient structure.
10977	* @param fStepping Whether we are single-stepping the guest in the
10978	* hypervisor debugger. Makes us ignore some of the reasons
10979	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10980	* if event dispatching took place.
10981	*/
10982	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10983	{
10984	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10985
10986	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10987	if (pVmxTransient->fIsNestedGuest)
10988	{
10989	RT_NOREF2(pVCpu, fStepping);
10990	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10991	return VINF_EM_RESCHEDULE_REM;
10992	}
10993	#endif
10994
10995	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10996	PGMRZDynMapFlushAutoSet(pVCpu);
10997	#endif
10998
10999	/*
11000	* Check and process force flag actions, some of which might require us to go back to ring-3.
11001	*/
11002	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11003	if (rcStrict == VINF_SUCCESS)
11004	{ /* FFs don't get set all the time. */ }
11005	else
11006	return rcStrict;
11007
11008	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11009	/*
11010	* Switch to the nested-guest VMCS as we may have transitioned into executing
11011	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
11012	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
11013	*
11014	* We do this as late as possible to minimize (though not completely remove)
11015	* clearing/loading VMCS again due to premature trips to ring-3 above.
11016	*/
11017	if (pVmxTransient->fIsNestedGuest)
11018	{
11019	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11020	{
11021	/*
11022	* Ensure we have synced everything from the guest VMCS and also flag that
11023	* that we need to export the full (nested) guest-CPU context to the
11024	* nested-guest VMCS.
11025	*/
11026	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11027	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
11028
11029	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
11030	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11031	if (RT_LIKELY(rc == VINF_SUCCESS))
11032	{
11033	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
11034	ASMSetFlags(fEFlags);
11035	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
11036
11037	/*
11038	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
11039	* flag that we need to update the host MSR values there. Even if we decide
11040	* in the future to share the VM-exit MSR-store area page with the guest,
11041	* if its content differs, we would have to update the host MSRs anyway.
11042	*/
11043	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
11044	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
11045	}
11046	else
11047	{
11048	ASMSetFlags(fEFlags);
11049	return rc;
11050	}
11051	}
11052
11053	/*
11054	* Merge guest VMCS controls with the nested-guest VMCS controls.
11055	*
11056	* Even if we have not executed the guest prior to this (e.g. when resuming
11057	* from a saved state), we should be okay with merging controls as we
11058	* initialize the guest VMCS controls as part of VM setup phase.
11059	*/
11060	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
11061	{
11062	int rc = hmR0VmxMergeVmcsNested(pVCpu);
11063	AssertRCReturn(rc, rc);
11064	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11065	}
11066	}
11067	#endif
11068
11069	/*
11070	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
11071	*/
11072	PVM pVM = pVCpu->CTX_SUFF(pVM);
11073	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11074	&& hmR0VmxIsProcCtls2Set(pVCpu, pVmxTransient, VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11075	&& PDMHasApic(pVM))
11076	{
11077	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11078	AssertRCReturn(rc, rc);
11079	}
11080
11081	/*
11082	* Evaluate events to be injected into the guest.
11083	*
11084	* Events in TRPM can be injected without inspecting the guest state.
11085	* If any new events (interrupts/NMI) are pending currently, we try to set up the
11086	* guest to cause a VM-exit the next time they are ready to receive the event.
11087	*/
11088	if (TRPMHasTrap(pVCpu))
11089	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11090
11091	uint32_t fIntrState;
11092	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11093
11094	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11095	/*
11096	* While evaluating pending events if something failed (unlikely) or if we were
11097	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11098	*/
11099	if (rcStrict != VINF_SUCCESS)
11100	return rcStrict;
11101	if ( pVmxTransient->fIsNestedGuest
11102	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11103	return VINF_VMX_VMEXIT;
11104	#else
11105	Assert(rcStrict == VINF_SUCCESS);
11106	#endif
11107
11108	/*
11109	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11110	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11111	* also result in triple-faulting the VM.
11112	*
11113	* The above does not apply when executing a nested-guest (since unrestricted guest execution
11114	* is a requirement) regardless doing it avoid duplicating code elsewhere.
11115	*/
11116	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11117	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11118	{ /* likely */ }
11119	else
11120	{
11121	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11122	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11123	return rcStrict;
11124	}
11125
11126	/*
11127	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
11128	* import CR3 themselves. We will need to update them here, as even as late as the above
11129	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11130	* the below force flags to be set.
11131	*/
11132	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11133	{
11134	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11135	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11136	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
11137	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11138	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11139	}
11140	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11141	{
11142	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11143	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11144	}
11145
11146	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11147	/* Paranoia. */
11148	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11149	#endif
11150
11151	/*
11152	* No longjmps to ring-3 from this point on!!!
11153	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11154	* This also disables flushing of the R0-logger instance (if any).
11155	*/
11156	VMMRZCallRing3Disable(pVCpu);
11157
11158	/*
11159	* Export the guest state bits.
11160	*
11161	* We cannot perform longjmps while loading the guest state because we do not preserve the
11162	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
11163	* CPU migration.
11164	*
11165	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11166	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
11167	*/
11168	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11169	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11170	{ /* likely */ }
11171	else
11172	{
11173	VMMRZCallRing3Enable(pVCpu);
11174	return rcStrict;
11175	}
11176
11177	/*
11178	* We disable interrupts so that we don't miss any interrupts that would flag preemption
11179	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11180	* preemption disabled for a while. Since this is purely to aid the
11181	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11182	* disable interrupt on NT.
11183	*
11184	* We need to check for force-flags that could've possible been altered since we last
11185	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11186	* see @bugref{6398}).
11187	*
11188	* We also check a couple of other force-flags as a last opportunity to get the EMT back
11189	* to ring-3 before executing guest code.
11190	*/
11191	pVmxTransient->fEFlags = ASMIntDisableFlags();
11192
11193	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
11194	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11195	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11196	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
11197	{
11198	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11199	{
11200	pVCpu->hm.s.Event.fPending = false;
11201
11202	/*
11203	* We've injected any pending events. This is really the point of no return (to ring-3).
11204	*
11205	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
11206	* returns from this function, so don't enable them here.
11207	*/
11208	return VINF_SUCCESS;
11209	}
11210
11211	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11212	rcStrict = VINF_EM_RAW_INTERRUPT;
11213	}
11214	else
11215	{
11216	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11217	rcStrict = VINF_EM_RAW_TO_R3;
11218	}
11219
11220	ASMSetFlags(pVmxTransient->fEFlags);
11221	VMMRZCallRing3Enable(pVCpu);
11222
11223	return rcStrict;
11224	}
11225
11226
11227	/**
11228	* Final preparations before executing guest code using hardware-assisted VMX.
11229	*
11230	* We can no longer get preempted to a different host CPU and there are no returns
11231	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11232	* failures), this function is not intended to fail sans unrecoverable hardware
11233	* errors.
11234	*
11235	* @param pVCpu The cross context virtual CPU structure.
11236	* @param pVmxTransient The VMX-transient structure.
11237	*
11238	* @remarks Called with preemption disabled.
11239	* @remarks No-long-jump zone!!!
11240	*/
11241	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11242	{
11243	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11244	Assert(VMMR0IsLogFlushDisabled(pVCpu));
11245	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11246	Assert(!pVCpu->hm.s.Event.fPending);
11247
11248	/*
11249	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11250	*/
11251	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11252	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11253
11254	PVM pVM = pVCpu->CTX_SUFF(pVM);
11255	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11256
11257	if (!CPUMIsGuestFPUStateActive(pVCpu))
11258	{
11259	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11260	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11261	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
11262	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11263	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11264	}
11265
11266	/*
11267	* Re-save the host state bits as we may've been preempted (only happens when
11268	* thread-context hooks are used or when the VM start function changes).
11269	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11270	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
11271	* see @bugref{8432}.
11272	*
11273	* This may also happen when switching to/from a nested-guest VMCS without leaving
11274	* ring-0.
11275	*/
11276	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11277	{
11278	int rc = hmR0VmxExportHostState(pVCpu);
11279	AssertRC(rc);
11280	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
11281	}
11282	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11283
11284	/*
11285	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
11286	*/
11287	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11288	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11289	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11290
11291	/*
11292	* Store status of the shared guest/host debug state at the time of VM-entry.
11293	*/
11294	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11295	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11296	{
11297	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11298	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11299	}
11300	else
11301	#endif
11302	{
11303	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11304	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11305	}
11306
11307	/*
11308	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11309	* more than one conditional check. The post-run side of our code shall determine
11310	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11311	*/
11312	if (pVmcsInfo->pbVirtApic)
11313	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11314
11315	/*
11316	* Update the host MSRs values in the VM-exit MSR-load area.
11317	*/
11318	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11319	{
11320	if (pVmcsInfo->cExitMsrLoad > 0)
11321	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11322	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11323	}
11324
11325	/*
11326	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11327	* VMX-preemption timer based on the next virtual sync clock deadline.
11328	*/
11329	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11330	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11331	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11332	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
11333	{
11334	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11335	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11336	}
11337
11338	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11339	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11340	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11341	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11342
11343	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11344
11345	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11346	as we're about to start executing the guest . */
11347
11348	/*
11349	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11350	*
11351	* This is done this late as updating the TSC offsetting/preemption timer above
11352	* figures out if we can skip intercepting RDTSCP by calculating the number of
11353	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
11354	*/
11355	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11356	{
11357	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11358	{
11359	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11360	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11361	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11362	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11363	true /* fSetReadWrite /, true / fUpdateHostMsr */);
11364	AssertRC(rc);
11365	}
11366	else
11367	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11368	}
11369
11370	#ifdef VBOX_STRICT
11371	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11372	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11373	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11374	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
11375	#endif
11376
11377	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11378	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11379	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11380	* see @bugref{9180#c54}. */
11381	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11382	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11383	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11384	#endif
11385	}
11386
11387
11388	/**
11389	* First C routine invoked after running guest code using hardware-assisted VMX.
11390	*
11391	* @param pVCpu The cross context virtual CPU structure.
11392	* @param pVmxTransient The VMX-transient structure.
11393	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11394	*
11395	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
11396	*
11397	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
11398	* unconditionally when it is safe to do so.
11399	*/
11400	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11401	{
11402	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11403
11404	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11405	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11406	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11407	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11408	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11409	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11410
11411	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11412	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11413	{
11414	uint64_t uGstTsc;
11415	if (!pVmxTransient->fIsNestedGuest)
11416	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11417	else
11418	{
11419	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11420	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11421	}
11422	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11423	}
11424
11425	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11426	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11427	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11428
11429	#if HC_ARCH_BITS == 64
11430	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11431	#endif
11432	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11433	/* The 64-on-32 switcher maintains VMCS-launch state on its own
11434	and we need to leave it alone here. */
11435	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11436	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11437	#else
11438	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11439	#endif
11440	#ifdef VBOX_STRICT
11441	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11442	#endif
11443	Assert(!ASMIntAreEnabled());
11444	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11445	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11446
11447	/*
11448	* Save the basic VM-exit reason and check if the VM-entry failed.
11449	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11450	*/
11451	uint32_t uExitReason;
11452	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11453	AssertRC(rc);
11454	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11455	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11456
11457	/*
11458	* Check if VMLAUNCH/VMRESUME succeeded.
11459	* If this failed, we cause a guru meditation and cease further execution.
11460	*
11461	* However, if we are executing a nested-guest we might fail if we use the
11462	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11463	*/
11464	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11465	{
11466	/*
11467	* Update the VM-exit history array here even if the VM-entry failed due to:
11468	* - Invalid guest state.
11469	* - MSR loading.
11470	* - Machine-check event.
11471	*
11472	* In any of the above cases we will still have a "valid" VM-exit reason
11473	* despite @a fVMEntryFailed being false.
11474	*
11475	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11476	*
11477	* Note! We don't have CS or RIP at this point. Will probably address that later
11478	* by amending the history entry added here.
11479	*/
11480	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11481	UINT64_MAX, uHostTsc);
11482
11483	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11484	{
11485	VMMRZCallRing3Enable(pVCpu);
11486
11487	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11488	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11489
11490	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11491	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11492	AssertRC(rc);
11493	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11494	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11495	AssertRC(rc);
11496	#else
11497	/*
11498	* Import the guest-interruptibility state always as we need it while evaluating
11499	* injecting events on re-entry.
11500	*
11501	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11502	* checking for real-mode while exporting the state because all bits that cause
11503	* mode changes wrt CR0 are intercepted.
11504	*/
11505	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11506	AssertRC(rc);
11507	#endif
11508
11509	/*
11510	* Sync the TPR shadow with our APIC state.
11511	*/
11512	if ( !pVmxTransient->fIsNestedGuest
11513	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11514	{
11515	Assert(pVmcsInfo->pbVirtApic);
11516	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11517	{
11518	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11519	AssertRC(rc);
11520	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11521	}
11522	}
11523
11524	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11525	return;
11526	}
11527	}
11528	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11529	else if (pVmxTransient->fIsNestedGuest)
11530	{
11531	# if 0
11532	/*
11533	* Copy the VM-instruction error field to the guest VMCS.
11534	*/
11535	/** @todo NSTVMX: Verify we're using the fast path. */
11536	uint32_t u32RoVmInstrError;
11537	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
11538	AssertRCReturn(rc, rc);
11539	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
11540	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
11541	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
11542	# else
11543	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11544	# endif
11545	}
11546	#endif
11547	else
11548	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11549
11550	VMMRZCallRing3Enable(pVCpu);
11551	}
11552
11553
11554	/**
11555	* Runs the guest code using hardware-assisted VMX the normal way.
11556	*
11557	* @returns VBox status code.
11558	* @param pVCpu The cross context virtual CPU structure.
11559	* @param pcLoops Pointer to the number of executed loops.
11560	*/
11561	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11562	{
11563	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11564	Assert(pcLoops);
11565	Assert(*pcLoops <= cMaxResumeLoops);
11566
11567	VMXTRANSIENT VmxTransient;
11568	RT_ZERO(VmxTransient);
11569	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11570
11571	/* Paranoia. */
11572	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11573	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11574
11575	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11576	for (;;)
11577	{
11578	Assert(!HMR0SuspendPending());
11579	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11580	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11581
11582	/*
11583	* Preparatory work for running nested-guest code, this may force us to
11584	* return to ring-3.
11585	*
11586	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11587	*/
11588	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11589	if (rcStrict != VINF_SUCCESS)
11590	break;
11591
11592	/* Interrupts are disabled at this point! */
11593	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11594	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11595	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11596	/* Interrupts are re-enabled at this point! */
11597
11598	/*
11599	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11600	*/
11601	if (RT_SUCCESS(rcRun))
11602	{ /* very likely */ }
11603	else
11604	{
11605	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11606	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11607	return rcRun;
11608	}
11609
11610	/*
11611	* Profile the VM-exit.
11612	*/
11613	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11615	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11616	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11617	HMVMX_START_EXIT_DISPATCH_PROF();
11618
11619	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11620
11621	/*
11622	* Handle the VM-exit.
11623	*/
11624	#ifdef HMVMX_USE_FUNCTION_TABLE
11625	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11626	#else
11627	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11628	#endif
11629	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11630	if (rcStrict == VINF_SUCCESS)
11631	{
11632	if (++(*pcLoops) <= cMaxResumeLoops)
11633	continue;
11634	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11635	rcStrict = VINF_EM_RAW_INTERRUPT;
11636	}
11637	break;
11638	}
11639
11640	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11641	return rcStrict;
11642	}
11643
11644	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11645	/**
11646	* Runs the nested-guest code using hardware-assisted VMX.
11647	*
11648	* @returns VBox status code.
11649	* @param pVCpu The cross context virtual CPU structure.
11650	* @param pcLoops Pointer to the number of executed loops.
11651	*
11652	* @sa hmR0VmxRunGuestCodeNormal.
11653	*/
11654	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11655	{
11656	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11657	Assert(pcLoops);
11658	Assert(*pcLoops <= cMaxResumeLoops);
11659
11660	VMXTRANSIENT VmxTransient;
11661	RT_ZERO(VmxTransient);
11662	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11663	VmxTransient.fIsNestedGuest = true;
11664
11665	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11666	for (;;)
11667	{
11668	Assert(!HMR0SuspendPending());
11669	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11670	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11671
11672	/*
11673	* Preparatory work for running guest code, this may force us to
11674	* return to ring-3.
11675	*
11676	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11677	*/
11678	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11679	if (rcStrict != VINF_SUCCESS)
11680	break;
11681
11682	/* Interrupts are disabled at this point! */
11683	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11684	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11685	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11686	/* Interrupts are re-enabled at this point! */
11687
11688	/*
11689	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11690	*/
11691	if (RT_SUCCESS(rcRun))
11692	{ /* very likely */ }
11693	else
11694	{
11695	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11696	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11697	return rcRun;
11698	}
11699
11700	/*
11701	* Profile the VM-exit.
11702	*/
11703	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11705	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11706	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11707	HMVMX_START_EXIT_DISPATCH_PROF();
11708
11709	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11710
11711	/*
11712	* Handle the VM-exit.
11713	*/
11714	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11715	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11716	if ( rcStrict == VINF_SUCCESS
11717	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11718	{
11719	if (++(*pcLoops) <= cMaxResumeLoops)
11720	continue;
11721	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11722	rcStrict = VINF_EM_RAW_INTERRUPT;
11723	}
11724	break;
11725	}
11726
11727	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11728	return rcStrict;
11729	}
11730	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11731
11732
11733	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11734	* probes.
11735	*
11736	* The following few functions and associated structure contains the bloat
11737	* necessary for providing detailed debug events and dtrace probes as well as
11738	* reliable host side single stepping. This works on the principle of
11739	* "subclassing" the normal execution loop and workers. We replace the loop
11740	* method completely and override selected helpers to add necessary adjustments
11741	* to their core operation.
11742	*
11743	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11744	* any performance for debug and analysis features.
11745	*
11746	* @{
11747	*/
11748
11749	/**
11750	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11751	* the debug run loop.
11752	*/
11753	typedef struct VMXRUNDBGSTATE
11754	{
11755	/** The RIP we started executing at. This is for detecting that we stepped. */
11756	uint64_t uRipStart;
11757	/** The CS we started executing with. */
11758	uint16_t uCsStart;
11759
11760	/** Whether we've actually modified the 1st execution control field. */
11761	bool fModifiedProcCtls : 1;
11762	/** Whether we've actually modified the 2nd execution control field. */
11763	bool fModifiedProcCtls2 : 1;
11764	/** Whether we've actually modified the exception bitmap. */
11765	bool fModifiedXcptBitmap : 1;
11766
11767	/** We desire the modified the CR0 mask to be cleared. */
11768	bool fClearCr0Mask : 1;
11769	/** We desire the modified the CR4 mask to be cleared. */
11770	bool fClearCr4Mask : 1;
11771	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11772	uint32_t fCpe1Extra;
11773	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11774	uint32_t fCpe1Unwanted;
11775	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11776	uint32_t fCpe2Extra;
11777	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11778	uint32_t bmXcptExtra;
11779	/** The sequence number of the Dtrace provider settings the state was
11780	* configured against. */
11781	uint32_t uDtraceSettingsSeqNo;
11782	/** VM-exits to check (one bit per VM-exit). */
11783	uint32_t bmExitsToCheck[3];
11784
11785	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11786	uint32_t fProcCtlsInitial;
11787	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11788	uint32_t fProcCtls2Initial;
11789	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11790	uint32_t bmXcptInitial;
11791	} VMXRUNDBGSTATE;
11792	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11793	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11794
11795
11796	/**
11797	* Initializes the VMXRUNDBGSTATE structure.
11798	*
11799	* @param pVCpu The cross context virtual CPU structure of the
11800	* calling EMT.
11801	* @param pVmxTransient The VMX-transient structure.
11802	* @param pDbgState The debug state to initialize.
11803	*/
11804	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11805	{
11806	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11807	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11808
11809	pDbgState->fModifiedProcCtls = false;
11810	pDbgState->fModifiedProcCtls2 = false;
11811	pDbgState->fModifiedXcptBitmap = false;
11812	pDbgState->fClearCr0Mask = false;
11813	pDbgState->fClearCr4Mask = false;
11814	pDbgState->fCpe1Extra = 0;
11815	pDbgState->fCpe1Unwanted = 0;
11816	pDbgState->fCpe2Extra = 0;
11817	pDbgState->bmXcptExtra = 0;
11818	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11819	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11820	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11821	}
11822
11823
11824	/**
11825	* Updates the VMSC fields with changes requested by @a pDbgState.
11826	*
11827	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11828	* immediately before executing guest code, i.e. when interrupts are disabled.
11829	* We don't check status codes here as we cannot easily assert or return in the
11830	* latter case.
11831	*
11832	* @param pVCpu The cross context virtual CPU structure.
11833	* @param pVmxTransient The VMX-transient structure.
11834	* @param pDbgState The debug state.
11835	*/
11836	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11837	{
11838	/*
11839	* Ensure desired flags in VMCS control fields are set.
11840	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11841	*
11842	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11843	* there should be no stale data in pCtx at this point.
11844	*/
11845	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11846	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11847	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11848	{
11849	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11850	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11851	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11852	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11853	pDbgState->fModifiedProcCtls = true;
11854	}
11855
11856	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11857	{
11858	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11859	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11860	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11861	pDbgState->fModifiedProcCtls2 = true;
11862	}
11863
11864	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11865	{
11866	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11867	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11868	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11869	pDbgState->fModifiedXcptBitmap = true;
11870	}
11871
11872	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11873	{
11874	pVmcsInfo->u64Cr0Mask = 0;
11875	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11876	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11877	}
11878
11879	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11880	{
11881	pVmcsInfo->u64Cr4Mask = 0;
11882	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11883	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11884	}
11885
11886	NOREF(pVCpu);
11887	}
11888
11889
11890	/**
11891	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11892	* re-entry next time around.
11893	*
11894	* @returns Strict VBox status code (i.e. informational status codes too).
11895	* @param pVCpu The cross context virtual CPU structure.
11896	* @param pVmxTransient The VMX-transient structure.
11897	* @param pDbgState The debug state.
11898	* @param rcStrict The return code from executing the guest using single
11899	* stepping.
11900	*/
11901	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11902	VBOXSTRICTRC rcStrict)
11903	{
11904	/*
11905	* Restore VM-exit control settings as we may not reenter this function the
11906	* next time around.
11907	*/
11908	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11909
11910	/* We reload the initial value, trigger what we can of recalculations the
11911	next time around. From the looks of things, that's all that's required atm. */
11912	if (pDbgState->fModifiedProcCtls)
11913	{
11914	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11915	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11916	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11917	AssertRCReturn(rc2, rc2);
11918	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11919	}
11920
11921	/* We're currently the only ones messing with this one, so just restore the
11922	cached value and reload the field. */
11923	if ( pDbgState->fModifiedProcCtls2
11924	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11925	{
11926	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11927	AssertRCReturn(rc2, rc2);
11928	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11929	}
11930
11931	/* If we've modified the exception bitmap, we restore it and trigger
11932	reloading and partial recalculation the next time around. */
11933	if (pDbgState->fModifiedXcptBitmap)
11934	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11935
11936	return rcStrict;
11937	}
11938
11939
11940	/**
11941	* Configures VM-exit controls for current DBGF and DTrace settings.
11942	*
11943	* This updates @a pDbgState and the VMCS execution control fields to reflect
11944	* the necessary VM-exits demanded by DBGF and DTrace.
11945	*
11946	* @param pVCpu The cross context virtual CPU structure.
11947	* @param pVmxTransient The VMX-transient structure. May update
11948	* fUpdatedTscOffsettingAndPreemptTimer.
11949	* @param pDbgState The debug state.
11950	*/
11951	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11952	{
11953	/*
11954	* Take down the dtrace serial number so we can spot changes.
11955	*/
11956	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11957	ASMCompilerBarrier();
11958
11959	/*
11960	* We'll rebuild most of the middle block of data members (holding the
11961	* current settings) as we go along here, so start by clearing it all.
11962	*/
11963	pDbgState->bmXcptExtra = 0;
11964	pDbgState->fCpe1Extra = 0;
11965	pDbgState->fCpe1Unwanted = 0;
11966	pDbgState->fCpe2Extra = 0;
11967	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11968	pDbgState->bmExitsToCheck[i] = 0;
11969
11970	/*
11971	* Software interrupts (INT XXh) - no idea how to trigger these...
11972	*/
11973	PVM pVM = pVCpu->CTX_SUFF(pVM);
11974	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11975	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11976	{
11977	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11978	}
11979
11980	/*
11981	* INT3 breakpoints - triggered by #BP exceptions.
11982	*/
11983	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11984	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11985
11986	/*
11987	* Exception bitmap and XCPT events+probes.
11988	*/
11989	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11990	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11991	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11992
11993	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11994	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11995	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11996	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11997	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11998	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11999	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
12000	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
12001	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
12002	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
12003	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
12004	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
12005	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
12006	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
12007	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
12008	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
12009	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
12010	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
12011
12012	if (pDbgState->bmXcptExtra)
12013	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12014
12015	/*
12016	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12017	*
12018	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12019	* So, when adding/changing/removing please don't forget to update it.
12020	*
12021	* Some of the macros are picking up local variables to save horizontal space,
12022	* (being able to see it in a table is the lesser evil here).
12023	*/
12024	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12025	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12026	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12027	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12028	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12029	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12030	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12031	} else do { } while (0)
12032	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12033	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12034	{ \
12035	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
12036	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12037	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12038	} else do { } while (0)
12039	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12040	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12041	{ \
12042	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
12043	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12044	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12045	} else do { } while (0)
12046	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12047	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12048	{ \
12049	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
12050	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12051	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12052	} else do { } while (0)
12053
12054	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12055	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12056	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12057	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12058	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12059
12060	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12061	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12062	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12063	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12064	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12065	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12066	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12067	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12068	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12069	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12070	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12071	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12072	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12073	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12074	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12075	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12076	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12077	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12078	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12079	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12080	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12081	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12082	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12083	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12084	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12085	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12086	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12087	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12088	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12089	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12090	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12091	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12092	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12093	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12094	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12095	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12096
12097	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12098	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12099	{
12100	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
12101	\| CPUMCTX_EXTRN_APIC_TPR);
12102	AssertRC(rc);
12103
12104	#if 0 /** @todo fix me */
12105	pDbgState->fClearCr0Mask = true;
12106	pDbgState->fClearCr4Mask = true;
12107	#endif
12108	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12109	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
12110	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12111	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
12112	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12113	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12114	require clearing here and in the loop if we start using it. */
12115	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12116	}
12117	else
12118	{
12119	if (pDbgState->fClearCr0Mask)
12120	{
12121	pDbgState->fClearCr0Mask = false;
12122	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12123	}
12124	if (pDbgState->fClearCr4Mask)
12125	{
12126	pDbgState->fClearCr4Mask = false;
12127	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12128	}
12129	}
12130	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12131	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12132
12133	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12134	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12135	{
12136	/** @todo later, need to fix handler as it assumes this won't usually happen. */
12137	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12138	}
12139	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12140	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12141
12142	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12143	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12144	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12145	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12146	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12147	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12148	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12149	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12150	#if 0 /** @todo too slow, fix handler. */
12151	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12152	#endif
12153	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12154
12155	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12156	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12157	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12158	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12159	{
12160	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12161	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12162	}
12163	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12164	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12165	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12166	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12167
12168	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12169	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
12170	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12171	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
12172	{
12173	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12174	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12175	}
12176	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12177	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12178	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12179	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12180
12181	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12182	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12183	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12184	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12185	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12186	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12187	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12188	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12189	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12190	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12191	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12192	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12193	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12194	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12195	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12196	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12197	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12198	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12199	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12200	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12201	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12202	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12203
12204	#undef IS_EITHER_ENABLED
12205	#undef SET_ONLY_XBM_IF_EITHER_EN
12206	#undef SET_CPE1_XBM_IF_EITHER_EN
12207	#undef SET_CPEU_XBM_IF_EITHER_EN
12208	#undef SET_CPE2_XBM_IF_EITHER_EN
12209
12210	/*
12211	* Sanitize the control stuff.
12212	*/
12213	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12214	if (pDbgState->fCpe2Extra)
12215	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12216	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12217	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12218	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12219	{
12220	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12221	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12222	}
12223
12224	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12225	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12226	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12227	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12228	}
12229
12230
12231	/**
12232	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
12233	* appropriate.
12234	*
12235	* The caller has checked the VM-exit against the
12236	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12237	* already, so we don't have to do that either.
12238	*
12239	* @returns Strict VBox status code (i.e. informational status codes too).
12240	* @param pVCpu The cross context virtual CPU structure.
12241	* @param pVmxTransient The VMX-transient structure.
12242	* @param uExitReason The VM-exit reason.
12243	*
12244	* @remarks The name of this function is displayed by dtrace, so keep it short
12245	* and to the point. No longer than 33 chars long, please.
12246	*/
12247	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12248	{
12249	/*
12250	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12251	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
12252	*
12253	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12254	* does. Must add/change/remove both places. Same ordering, please.
12255	*
12256	* Added/removed events must also be reflected in the next section
12257	* where we dispatch dtrace events.
12258	*/
12259	bool fDtrace1 = false;
12260	bool fDtrace2 = false;
12261	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12262	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12263	uint32_t uEventArg = 0;
12264	#define SET_EXIT(a_EventSubName) \
12265	do { \
12266	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12267	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12268	} while (0)
12269	#define SET_BOTH(a_EventSubName) \
12270	do { \
12271	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12272	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12273	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12274	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12275	} while (0)
12276	switch (uExitReason)
12277	{
12278	case VMX_EXIT_MTF:
12279	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12280
12281	case VMX_EXIT_XCPT_OR_NMI:
12282	{
12283	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12284	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12285	{
12286	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12287	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12288	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12289	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12290	{
12291	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12292	{
12293	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12294	uEventArg = pVmxTransient->uExitIntErrorCode;
12295	}
12296	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12297	switch (enmEvent1)
12298	{
12299	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12300	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12301	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12302	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12303	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12304	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12305	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12306	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12307	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12308	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12309	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12310	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12311	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12312	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12313	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12314	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12315	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12316	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12317	default: break;
12318	}
12319	}
12320	else
12321	AssertFailed();
12322	break;
12323
12324	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12325	uEventArg = idxVector;
12326	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12327	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12328	break;
12329	}
12330	break;
12331	}
12332
12333	case VMX_EXIT_TRIPLE_FAULT:
12334	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12335	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12336	break;
12337	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12338	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12339	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12340	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12341	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12342
12343	/* Instruction specific VM-exits: */
12344	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12345	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12346	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12347	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12348	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12349	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12350	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12351	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12352	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12353	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12354	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12355	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12356	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12357	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12358	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12359	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12360	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12361	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12362	case VMX_EXIT_MOV_CRX:
12363	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12364	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12365	SET_BOTH(CRX_READ);
12366	else
12367	SET_BOTH(CRX_WRITE);
12368	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12369	break;
12370	case VMX_EXIT_MOV_DRX:
12371	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12372	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12373	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12374	SET_BOTH(DRX_READ);
12375	else
12376	SET_BOTH(DRX_WRITE);
12377	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12378	break;
12379	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12380	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12381	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12382	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12383	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12384	case VMX_EXIT_GDTR_IDTR_ACCESS:
12385	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12386	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12387	{
12388	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12389	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12390	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12391	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12392	}
12393	break;
12394
12395	case VMX_EXIT_LDTR_TR_ACCESS:
12396	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12397	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12398	{
12399	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12400	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12401	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12402	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12403	}
12404	break;
12405
12406	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12407	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12408	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12409	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12410	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12411	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12412	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12413	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12414	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12415	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12416	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12417
12418	/* Events that aren't relevant at this point. */
12419	case VMX_EXIT_EXT_INT:
12420	case VMX_EXIT_INT_WINDOW:
12421	case VMX_EXIT_NMI_WINDOW:
12422	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12423	case VMX_EXIT_PREEMPT_TIMER:
12424	case VMX_EXIT_IO_INSTR:
12425	break;
12426
12427	/* Errors and unexpected events. */
12428	case VMX_EXIT_INIT_SIGNAL:
12429	case VMX_EXIT_SIPI:
12430	case VMX_EXIT_IO_SMI:
12431	case VMX_EXIT_SMI:
12432	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12433	case VMX_EXIT_ERR_MSR_LOAD:
12434	case VMX_EXIT_ERR_MACHINE_CHECK:
12435	case VMX_EXIT_PML_FULL:
12436	case VMX_EXIT_VIRTUALIZED_EOI:
12437	break;
12438
12439	default:
12440	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12441	break;
12442	}
12443	#undef SET_BOTH
12444	#undef SET_EXIT
12445
12446	/*
12447	* Dtrace tracepoints go first. We do them here at once so we don't
12448	* have to copy the guest state saving and stuff a few dozen times.
12449	* Down side is that we've got to repeat the switch, though this time
12450	* we use enmEvent since the probes are a subset of what DBGF does.
12451	*/
12452	if (fDtrace1 \|\| fDtrace2)
12453	{
12454	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12455	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12456	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12457	switch (enmEvent1)
12458	{
12459	/** @todo consider which extra parameters would be helpful for each probe. */
12460	case DBGFEVENT_END: break;
12461	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12462	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12463	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12464	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12465	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12466	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12467	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12468	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12469	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12470	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12471	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12472	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12473	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12474	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12475	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12476	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12477	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12478	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12479	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12480	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12481	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12482	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12483	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12484	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12485	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12486	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12487	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12488	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12489	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12490	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12491	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12492	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12493	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12494	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12495	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12496	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12497	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12498	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12499	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12500	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12501	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12502	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12503	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12504	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12505	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12506	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12507	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12508	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12509	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12510	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12511	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12512	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12513	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12514	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12515	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12516	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12517	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12518	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12519	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12520	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12521	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12522	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12523	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12524	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12525	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12526	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12527	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12528	}
12529	switch (enmEvent2)
12530	{
12531	/** @todo consider which extra parameters would be helpful for each probe. */
12532	case DBGFEVENT_END: break;
12533	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12534	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12535	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12536	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12537	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12538	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12539	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12540	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12541	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12542	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12543	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12544	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12545	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12546	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12547	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12548	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12549	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12550	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12551	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12552	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12553	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12554	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12555	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12556	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12557	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12558	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12559	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12560	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12561	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12562	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12563	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12564	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12565	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12566	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12567	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12568	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12569	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12570	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12571	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12572	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12573	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12574	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12575	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12576	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12577	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12578	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12579	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12580	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12581	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12582	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12583	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12584	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12585	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12586	}
12587	}
12588
12589	/*
12590	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12591	* the DBGF call will do a full check).
12592	*
12593	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12594	* Note! If we have to events, we prioritize the first, i.e. the instruction
12595	* one, in order to avoid event nesting.
12596	*/
12597	PVM pVM = pVCpu->CTX_SUFF(pVM);
12598	if ( enmEvent1 != DBGFEVENT_END
12599	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12600	{
12601	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12602	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12603	if (rcStrict != VINF_SUCCESS)
12604	return rcStrict;
12605	}
12606	else if ( enmEvent2 != DBGFEVENT_END
12607	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12608	{
12609	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12610	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12611	if (rcStrict != VINF_SUCCESS)
12612	return rcStrict;
12613	}
12614
12615	return VINF_SUCCESS;
12616	}
12617
12618
12619	/**
12620	* Single-stepping VM-exit filtering.
12621	*
12622	* This is preprocessing the VM-exits and deciding whether we've gotten far
12623	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12624	* handling is performed.
12625	*
12626	* @returns Strict VBox status code (i.e. informational status codes too).
12627	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12628	* @param pVmxTransient The VMX-transient structure.
12629	* @param pDbgState The debug state.
12630	*/
12631	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12632	{
12633	/*
12634	* Expensive (saves context) generic dtrace VM-exit probe.
12635	*/
12636	uint32_t const uExitReason = pVmxTransient->uExitReason;
12637	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12638	{ /* more likely */ }
12639	else
12640	{
12641	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12642	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12643	AssertRC(rc);
12644	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12645	}
12646
12647	/*
12648	* Check for host NMI, just to get that out of the way.
12649	*/
12650	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12651	{ /* normally likely */ }
12652	else
12653	{
12654	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12655	AssertRCReturn(rc2, rc2);
12656	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12657	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12658	return hmR0VmxExitHostNmi(pVCpu);
12659	}
12660
12661	/*
12662	* Check for single stepping event if we're stepping.
12663	*/
12664	if (pVCpu->hm.s.fSingleInstruction)
12665	{
12666	switch (uExitReason)
12667	{
12668	case VMX_EXIT_MTF:
12669	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12670
12671	/* Various events: */
12672	case VMX_EXIT_XCPT_OR_NMI:
12673	case VMX_EXIT_EXT_INT:
12674	case VMX_EXIT_TRIPLE_FAULT:
12675	case VMX_EXIT_INT_WINDOW:
12676	case VMX_EXIT_NMI_WINDOW:
12677	case VMX_EXIT_TASK_SWITCH:
12678	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12679	case VMX_EXIT_APIC_ACCESS:
12680	case VMX_EXIT_EPT_VIOLATION:
12681	case VMX_EXIT_EPT_MISCONFIG:
12682	case VMX_EXIT_PREEMPT_TIMER:
12683
12684	/* Instruction specific VM-exits: */
12685	case VMX_EXIT_CPUID:
12686	case VMX_EXIT_GETSEC:
12687	case VMX_EXIT_HLT:
12688	case VMX_EXIT_INVD:
12689	case VMX_EXIT_INVLPG:
12690	case VMX_EXIT_RDPMC:
12691	case VMX_EXIT_RDTSC:
12692	case VMX_EXIT_RSM:
12693	case VMX_EXIT_VMCALL:
12694	case VMX_EXIT_VMCLEAR:
12695	case VMX_EXIT_VMLAUNCH:
12696	case VMX_EXIT_VMPTRLD:
12697	case VMX_EXIT_VMPTRST:
12698	case VMX_EXIT_VMREAD:
12699	case VMX_EXIT_VMRESUME:
12700	case VMX_EXIT_VMWRITE:
12701	case VMX_EXIT_VMXOFF:
12702	case VMX_EXIT_VMXON:
12703	case VMX_EXIT_MOV_CRX:
12704	case VMX_EXIT_MOV_DRX:
12705	case VMX_EXIT_IO_INSTR:
12706	case VMX_EXIT_RDMSR:
12707	case VMX_EXIT_WRMSR:
12708	case VMX_EXIT_MWAIT:
12709	case VMX_EXIT_MONITOR:
12710	case VMX_EXIT_PAUSE:
12711	case VMX_EXIT_GDTR_IDTR_ACCESS:
12712	case VMX_EXIT_LDTR_TR_ACCESS:
12713	case VMX_EXIT_INVEPT:
12714	case VMX_EXIT_RDTSCP:
12715	case VMX_EXIT_INVVPID:
12716	case VMX_EXIT_WBINVD:
12717	case VMX_EXIT_XSETBV:
12718	case VMX_EXIT_RDRAND:
12719	case VMX_EXIT_INVPCID:
12720	case VMX_EXIT_VMFUNC:
12721	case VMX_EXIT_RDSEED:
12722	case VMX_EXIT_XSAVES:
12723	case VMX_EXIT_XRSTORS:
12724	{
12725	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12726	AssertRCReturn(rc, rc);
12727	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12728	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12729	return VINF_EM_DBG_STEPPED;
12730	break;
12731	}
12732
12733	/* Errors and unexpected events: */
12734	case VMX_EXIT_INIT_SIGNAL:
12735	case VMX_EXIT_SIPI:
12736	case VMX_EXIT_IO_SMI:
12737	case VMX_EXIT_SMI:
12738	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12739	case VMX_EXIT_ERR_MSR_LOAD:
12740	case VMX_EXIT_ERR_MACHINE_CHECK:
12741	case VMX_EXIT_PML_FULL:
12742	case VMX_EXIT_VIRTUALIZED_EOI:
12743	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12744	break;
12745
12746	default:
12747	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12748	break;
12749	}
12750	}
12751
12752	/*
12753	* Check for debugger event breakpoints and dtrace probes.
12754	*/
12755	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12756	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12757	{
12758	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12759	if (rcStrict != VINF_SUCCESS)
12760	return rcStrict;
12761	}
12762
12763	/*
12764	* Normal processing.
12765	*/
12766	#ifdef HMVMX_USE_FUNCTION_TABLE
12767	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12768	#else
12769	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12770	#endif
12771	}
12772
12773
12774	/**
12775	* Single steps guest code using hardware-assisted VMX.
12776	*
12777	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12778	* but single-stepping through the hypervisor debugger.
12779	*
12780	* @returns Strict VBox status code (i.e. informational status codes too).
12781	* @param pVCpu The cross context virtual CPU structure.
12782	* @param pcLoops Pointer to the number of executed loops.
12783	*
12784	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12785	*/
12786	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12787	{
12788	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12789	Assert(pcLoops);
12790	Assert(*pcLoops <= cMaxResumeLoops);
12791
12792	VMXTRANSIENT VmxTransient;
12793	RT_ZERO(VmxTransient);
12794	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12795
12796	/* Set HMCPU indicators. */
12797	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12798	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12799	pVCpu->hm.s.fDebugWantRdTscExit = false;
12800	pVCpu->hm.s.fUsingDebugLoop = true;
12801
12802	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12803	VMXRUNDBGSTATE DbgState;
12804	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12805	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12806
12807	/*
12808	* The loop.
12809	*/
12810	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12811	for (;;)
12812	{
12813	Assert(!HMR0SuspendPending());
12814	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12815	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12816	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12817
12818	/* Set up VM-execution controls the next two can respond to. */
12819	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12820
12821	/*
12822	* Preparatory work for running guest code, this may force us to
12823	* return to ring-3.
12824	*
12825	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12826	*/
12827	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12828	if (rcStrict != VINF_SUCCESS)
12829	break;
12830
12831	/* Interrupts are disabled at this point! */
12832	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12833
12834	/* Override any obnoxious code in the above two calls. */
12835	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12836
12837	/*
12838	* Finally execute the guest.
12839	*/
12840	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12841
12842	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12843	/* Interrupts are re-enabled at this point! */
12844
12845	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12846	if (RT_SUCCESS(rcRun))
12847	{ /* very likely */ }
12848	else
12849	{
12850	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12851	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12852	return rcRun;
12853	}
12854
12855	/* Profile the VM-exit. */
12856	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12858	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12859	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12860	HMVMX_START_EXIT_DISPATCH_PROF();
12861
12862	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12863
12864	/*
12865	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12866	*/
12867	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12868	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12869	if (rcStrict != VINF_SUCCESS)
12870	break;
12871	if (++(*pcLoops) > cMaxResumeLoops)
12872	{
12873	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12874	rcStrict = VINF_EM_RAW_INTERRUPT;
12875	break;
12876	}
12877
12878	/*
12879	* Stepping: Did the RIP change, if so, consider it a single step.
12880	* Otherwise, make sure one of the TFs gets set.
12881	*/
12882	if (fStepping)
12883	{
12884	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12885	AssertRC(rc);
12886	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12887	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12888	{
12889	rcStrict = VINF_EM_DBG_STEPPED;
12890	break;
12891	}
12892	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12893	}
12894
12895	/*
12896	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12897	*/
12898	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12899	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12900	}
12901
12902	/*
12903	* Clear the X86_EFL_TF if necessary.
12904	*/
12905	if (pVCpu->hm.s.fClearTrapFlag)
12906	{
12907	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12908	AssertRC(rc);
12909	pVCpu->hm.s.fClearTrapFlag = false;
12910	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12911	}
12912	/** @todo there seems to be issues with the resume flag when the monitor trap
12913	* flag is pending without being used. Seen early in bios init when
12914	* accessing APIC page in protected mode. */
12915
12916	/*
12917	* Restore VM-exit control settings as we may not re-enter this function the
12918	* next time around.
12919	*/
12920	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12921
12922	/* Restore HMCPU indicators. */
12923	pVCpu->hm.s.fUsingDebugLoop = false;
12924	pVCpu->hm.s.fDebugWantRdTscExit = false;
12925	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12926
12927	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12928	return rcStrict;
12929	}
12930
12931
12932	/** @} */
12933
12934
12935	/**
12936	* Checks if any expensive dtrace probes are enabled and we should go to the
12937	* debug loop.
12938	*
12939	* @returns true if we should use debug loop, false if not.
12940	*/
12941	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12942	{
12943	/* It's probably faster to OR the raw 32-bit counter variables together.
12944	Since the variables are in an array and the probes are next to one
12945	another (more or less), we have good locality. So, better read
12946	eight-nine cache lines ever time and only have one conditional, than
12947	128+ conditionals, right? */
12948	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12949	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12950	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12951	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12952	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12953	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12954	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12955	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12956	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12957	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12958	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12959	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12960	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12961	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12962	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12963	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12964	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12965	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12966	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12967	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12968	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12969	) != 0
12970	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12971	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12972	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12973	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12974	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12975	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12976	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12977	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12978	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12979	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12980	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12981	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12982	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12983	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12984	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12985	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12986	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12987	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12988	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12989	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12990	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12991	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12992	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12993	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12994	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12995	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12996	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12997	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12998	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12999	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13000	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13001	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13002	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13003	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13004	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13005	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13006	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13007	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13008	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13009	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13010	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13011	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13012	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13013	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13014	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13015	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13016	) != 0
13017	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13018	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
13019	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13020	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13021	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13022	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
13023	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13024	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13025	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13026	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13027	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13028	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13029	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13030	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13031	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13032	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13033	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13034	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13035	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13036	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13037	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13038	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13039	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13040	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13041	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13042	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
13043	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
13044	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13045	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
13046	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13047	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13048	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13049	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13050	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13051	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13052	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13053	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13054	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13055	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13056	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13057	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13058	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13059	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13060	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13061	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13062	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13063	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13064	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13065	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13066	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13067	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13068	) != 0;
13069	}
13070
13071
13072	/**
13073	* Runs the guest using hardware-assisted VMX.
13074	*
13075	* @returns Strict VBox status code (i.e. informational status codes too).
13076	* @param pVCpu The cross context virtual CPU structure.
13077	*/
13078	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13079	{
13080	AssertPtr(pVCpu);
13081	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13082	Assert(VMMRZCallRing3IsEnabled(pVCpu));
13083	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13084	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13085
13086	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13087
13088	VBOXSTRICTRC rcStrict;
13089	uint32_t cLoops = 0;
13090	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13091	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13092	#else
13093	bool const fInNestedGuestMode = false;
13094	#endif
13095	if (!fInNestedGuestMode)
13096	{
13097	if ( !pVCpu->hm.s.fUseDebugLoop
13098	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
13099	&& !DBGFIsStepping(pVCpu)
13100	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13101	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13102	else
13103	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13104	}
13105	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13106	else
13107	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
13108
13109	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13110	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13111	#endif
13112
13113	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13114	switch (rcLoop)
13115	{
13116	case VINF_VMX_VMEXIT: rcStrict = VINF_SUCCESS; break;
13117	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13118	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13119	}
13120
13121	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13122	if (RT_FAILURE(rc2))
13123	{
13124	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13125	rcStrict = rc2;
13126	}
13127	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13128	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13129	return rcStrict;
13130	}
13131
13132
13133	#ifndef HMVMX_USE_FUNCTION_TABLE
13134	/**
13135	* Handles a guest VM-exit from hardware-assisted VMX execution.
13136	*
13137	* @returns Strict VBox status code (i.e. informational status codes too).
13138	* @param pVCpu The cross context virtual CPU structure.
13139	* @param pVmxTransient The VMX-transient structure.
13140	*/
13141	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13142	{
13143	#ifdef DEBUG_ramshankar
13144	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13145	do { \
13146	if (a_fSave != 0) \
13147	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
13148	VBOXSTRICTRC rcStrict = a_CallExpr; \
13149	if (a_fSave != 0) \
13150	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13151	return rcStrict; \
13152	} while (0)
13153	#else
13154	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13155	#endif
13156	uint32_t const uExitReason = pVmxTransient->uExitReason;
13157	switch (uExitReason)
13158	{
13159	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13160	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13161	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13162	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13163	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13164	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13165	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13166	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13167	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13168	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13169	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13170	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13171	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13172	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13173	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13174	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13175	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13176	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13177	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13178	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13179	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13180	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13181	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13182	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13183	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13184	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13185	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13186	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13187	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13188	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13189	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13190	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13191	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13192	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13193	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13194	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13195	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13196	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13197	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13198	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13199	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13200	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13201	#else
13202	case VMX_EXIT_VMCLEAR:
13203	case VMX_EXIT_VMLAUNCH:
13204	case VMX_EXIT_VMPTRLD:
13205	case VMX_EXIT_VMPTRST:
13206	case VMX_EXIT_VMREAD:
13207	case VMX_EXIT_VMRESUME:
13208	case VMX_EXIT_VMWRITE:
13209	case VMX_EXIT_VMXOFF:
13210	case VMX_EXIT_VMXON:
13211	case VMX_EXIT_INVVPID:
13212	case VMX_EXIT_INVEPT:
13213	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13214	#endif
13215
13216	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13217	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13218	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13219
13220	case VMX_EXIT_INIT_SIGNAL:
13221	case VMX_EXIT_SIPI:
13222	case VMX_EXIT_IO_SMI:
13223	case VMX_EXIT_SMI:
13224	case VMX_EXIT_ERR_MSR_LOAD:
13225	case VMX_EXIT_ERR_MACHINE_CHECK:
13226	case VMX_EXIT_PML_FULL:
13227	case VMX_EXIT_VIRTUALIZED_EOI:
13228	case VMX_EXIT_GDTR_IDTR_ACCESS:
13229	case VMX_EXIT_LDTR_TR_ACCESS:
13230	case VMX_EXIT_APIC_WRITE:
13231	case VMX_EXIT_RDRAND:
13232	case VMX_EXIT_RSM:
13233	case VMX_EXIT_VMFUNC:
13234	case VMX_EXIT_ENCLS:
13235	case VMX_EXIT_RDSEED:
13236	case VMX_EXIT_XSAVES:
13237	case VMX_EXIT_XRSTORS:
13238	case VMX_EXIT_UMWAIT:
13239	case VMX_EXIT_TPAUSE:
13240	default:
13241	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13242	}
13243	#undef VMEXIT_CALL_RET
13244	}
13245	#endif /* !HMVMX_USE_FUNCTION_TABLE */
13246
13247
13248	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13249	/**
13250	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13251	*
13252	* @returns Strict VBox status code (i.e. informational status codes too).
13253	* @param pVCpu The cross context virtual CPU structure.
13254	* @param pVmxTransient The VMX-transient structure.
13255	*/
13256	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13257	{
13258	uint32_t const uExitReason = pVmxTransient->uExitReason;
13259	switch (uExitReason)
13260	{
13261	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13262	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13263	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13264	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13265	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13266
13267	/*
13268	* We shouldn't direct host physical interrupts to the nested-guest.
13269	*/
13270	case VMX_EXIT_EXT_INT:
13271	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13272
13273	/*
13274	* Instructions that cause VM-exits unconditionally or the condition is
13275	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13276	* happens, it's guaranteed to be a nested-guest VM-exit).
13277	*
13278	* - Provides VM-exit instruction length ONLY.
13279	*/
13280	case VMX_EXIT_CPUID: /* Unconditional. */
13281	case VMX_EXIT_VMCALL:
13282	case VMX_EXIT_GETSEC:
13283	case VMX_EXIT_INVD:
13284	case VMX_EXIT_XSETBV:
13285	case VMX_EXIT_VMLAUNCH:
13286	case VMX_EXIT_VMRESUME:
13287	case VMX_EXIT_VMXOFF:
13288	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13289	case VMX_EXIT_VMFUNC:
13290	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13291
13292	/*
13293	* Instructions that cause VM-exits unconditionally or the condition is
13294	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13295	* happens, it's guaranteed to be a nested-guest VM-exit).
13296	*
13297	* - Provides VM-exit instruction length.
13298	* - Provides VM-exit information.
13299	* - Optionally provides Exit qualification.
13300	*
13301	* Since Exit qualification is 0 for all VM-exits where it is not
13302	* applicable, reading and passing it to the guest should produce
13303	* defined behavior.
13304	*
13305	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
13306	*/
13307	case VMX_EXIT_INVEPT: /* Unconditional. */
13308	case VMX_EXIT_INVVPID:
13309	case VMX_EXIT_VMCLEAR:
13310	case VMX_EXIT_VMPTRLD:
13311	case VMX_EXIT_VMPTRST:
13312	case VMX_EXIT_VMXON:
13313	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13314	case VMX_EXIT_LDTR_TR_ACCESS:
13315	case VMX_EXIT_RDRAND:
13316	case VMX_EXIT_RDSEED:
13317	case VMX_EXIT_XSAVES:
13318	case VMX_EXIT_XRSTORS:
13319	case VMX_EXIT_UMWAIT:
13320	case VMX_EXIT_TPAUSE:
13321	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13322
13323	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13324	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13325	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13326	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13327	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13328	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13329	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13330	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13331	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13332	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13333	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13334	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13335	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13336	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13337	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13338	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13339	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13340	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13341	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13342
13343	case VMX_EXIT_PREEMPT_TIMER:
13344	{
13345	/** @todo NSTVMX: Preempt timer. */
13346	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13347	}
13348
13349	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13350	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13351
13352	case VMX_EXIT_VMREAD:
13353	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13354
13355	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13356	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13357
13358	case VMX_EXIT_INIT_SIGNAL:
13359	case VMX_EXIT_SIPI:
13360	case VMX_EXIT_IO_SMI:
13361	case VMX_EXIT_SMI:
13362	case VMX_EXIT_ERR_MSR_LOAD:
13363	case VMX_EXIT_ERR_MACHINE_CHECK:
13364	case VMX_EXIT_PML_FULL:
13365	case VMX_EXIT_RSM:
13366	default:
13367	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13368	}
13369	}
13370	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13371
13372
13373	#ifdef VBOX_STRICT
13374	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13375	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13376	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13377
13378	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13379	do { \
13380	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13381	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13382	} while (0)
13383
13384	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13385	do { \
13386	AssertPtr((a_pVCpu)); \
13387	AssertPtr((a_pVmxTransient)); \
13388	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13389	Assert((a_pVmxTransient)->pVmcsInfo); \
13390	Assert(ASMIntAreEnabled()); \
13391	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13392	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13393	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
13394	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13395	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13396	HMVMX_ASSERT_PREEMPT_CPUID(); \
13397	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13398	} while (0)
13399
13400	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13401	do { \
13402	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13403	Assert((a_pVmxTransient)->fIsNestedGuest); \
13404	} while (0)
13405
13406	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13407	do { \
13408	Log4Func(("\n")); \
13409	} while (0)
13410	#else
13411	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13412	do { \
13413	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13414	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13415	} while (0)
13416
13417	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13418	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13419
13420	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13421	#endif
13422
13423
13424	/**
13425	* Advances the guest RIP by the specified number of bytes.
13426	*
13427	* @param pVCpu The cross context virtual CPU structure.
13428	* @param cbInstr Number of bytes to advance the RIP by.
13429	*
13430	* @remarks No-long-jump zone!!!
13431	*/
13432	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13433	{
13434	/* Advance the RIP. */
13435	pVCpu->cpum.GstCtx.rip += cbInstr;
13436	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13437
13438	/* Update interrupt inhibition. */
13439	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13440	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13441	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13442	}
13443
13444
13445	/**
13446	* Advances the guest RIP after reading it from the VMCS.
13447	*
13448	* @returns VBox status code, no informational status codes.
13449	* @param pVCpu The cross context virtual CPU structure.
13450	* @param pVmxTransient The VMX-transient structure.
13451	*
13452	* @remarks No-long-jump zone!!!
13453	*/
13454	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13455	{
13456	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13457	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13458	AssertRCReturn(rc, rc);
13459
13460	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13461	return VINF_SUCCESS;
13462	}
13463
13464
13465	/**
13466	* Handle a condition that occurred while delivering an event through the guest
13467	* IDT.
13468	*
13469	* @returns Strict VBox status code (i.e. informational status codes too).
13470	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13471	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13472	* to continue execution of the guest which will delivery the \#DF.
13473	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13474	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13475	*
13476	* @param pVCpu The cross context virtual CPU structure.
13477	* @param pVmxTransient The VMX-transient structure.
13478	*
13479	* @remarks No-long-jump zone!!!
13480	*/
13481	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13482	{
13483	/* Read the IDT vectoring info. and VM-exit interruption info. */
13484	{
13485	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13486	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13487	AssertRCReturn(rc, rc);
13488	}
13489
13490	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13491	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13492	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13493	{
13494	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13495	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13496
13497	/*
13498	* If the event was a software interrupt (generated with INT n) or a software exception
13499	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13500	* can handle the VM-exit and continue guest execution which will re-execute the
13501	* instruction rather than re-injecting the exception, as that can cause premature
13502	* trips to ring-3 before injection and involve TRPM which currently has no way of
13503	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13504	* the problem).
13505	*/
13506	IEMXCPTRAISE enmRaise;
13507	IEMXCPTRAISEINFO fRaiseInfo;
13508	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13509	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13510	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13511	{
13512	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13513	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13514	}
13515	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13516	{
13517	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13518	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13519	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13520
13521	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13522	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13523	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13524
13525	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13526
13527	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13528	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13529	{
13530	pVmxTransient->fVectoringPF = true;
13531	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13532	}
13533	}
13534	else
13535	{
13536	/*
13537	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13538	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13539	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13540	*/
13541	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13542	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13543	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13544	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13545	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13546	}
13547
13548	/*
13549	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13550	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13551	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13552	* subsequent VM-entry would fail.
13553	*
13554	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13555	*/
13556	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13557	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13558	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13559	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13560	&& CPUMIsGuestNmiBlocking(pVCpu))
13561	{
13562	CPUMSetGuestNmiBlocking(pVCpu, false);
13563	}
13564
13565	switch (enmRaise)
13566	{
13567	case IEMXCPTRAISE_CURRENT_XCPT:
13568	{
13569	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13570	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13571	Assert(rcStrict == VINF_SUCCESS);
13572	break;
13573	}
13574
13575	case IEMXCPTRAISE_PREV_EVENT:
13576	{
13577	uint32_t u32ErrCode;
13578	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13579	{
13580	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13581	AssertRCReturn(rc, rc);
13582	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13583	}
13584	else
13585	u32ErrCode = 0;
13586
13587	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13588	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13589	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13590	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13591
13592	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13593	pVCpu->hm.s.Event.u32ErrCode));
13594	Assert(rcStrict == VINF_SUCCESS);
13595	break;
13596	}
13597
13598	case IEMXCPTRAISE_REEXEC_INSTR:
13599	Assert(rcStrict == VINF_SUCCESS);
13600	break;
13601
13602	case IEMXCPTRAISE_DOUBLE_FAULT:
13603	{
13604	/*
13605	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13606	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13607	*/
13608	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13609	{
13610	pVmxTransient->fVectoringDoublePF = true;
13611	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13612	pVCpu->cpum.GstCtx.cr2));
13613	rcStrict = VINF_SUCCESS;
13614	}
13615	else
13616	{
13617	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13618	hmR0VmxSetPendingXcptDF(pVCpu);
13619	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13620	uIdtVector, uExitVector));
13621	rcStrict = VINF_HM_DOUBLE_FAULT;
13622	}
13623	break;
13624	}
13625
13626	case IEMXCPTRAISE_TRIPLE_FAULT:
13627	{
13628	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13629	rcStrict = VINF_EM_RESET;
13630	break;
13631	}
13632
13633	case IEMXCPTRAISE_CPU_HANG:
13634	{
13635	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13636	rcStrict = VERR_EM_GUEST_CPU_HANG;
13637	break;
13638	}
13639
13640	default:
13641	{
13642	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13643	rcStrict = VERR_VMX_IPE_2;
13644	break;
13645	}
13646	}
13647	}
13648	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13649	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13650	&& uExitVector != X86_XCPT_DF
13651	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13652	{
13653	/*
13654	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13655	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13656	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13657	*/
13658	CPUMSetGuestNmiBlocking(pVCpu, true);
13659	Log4Func(("Set NMI blocking. fValid=%RTbool uExitReason=%u\n", VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo),
13660	pVmxTransient->uExitReason));
13661	}
13662
13663	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13664	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13665	return rcStrict;
13666	}
13667
13668
13669	/** @name VM-exit handlers.
13670	* @{
13671	*/
13672	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13673	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13674	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13675
13676	/**
13677	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13678	*/
13679	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13680	{
13681	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13683	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13684	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13685	return VINF_SUCCESS;
13686	return VINF_EM_RAW_INTERRUPT;
13687	}
13688
13689
13690	/**
13691	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
13692	* VM-exit.
13693	*/
13694	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13695	{
13696	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13697	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13698
13699	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13700	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13701	AssertRCReturn(rc, rc);
13702
13703	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13704	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13705	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13706	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13707
13708	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13709	{
13710	/*
13711	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13712	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13713	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13714	*
13715	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13716	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13717	*/
13718	return hmR0VmxExitHostNmi(pVCpu);
13719	}
13720
13721	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13722	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13723	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
13724	{ /* likely */ }
13725	else
13726	{
13727	if (rcStrict == VINF_HM_DOUBLE_FAULT)
13728	rcStrict = VINF_SUCCESS;
13729	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13730	return rcStrict;
13731	}
13732
13733	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13734	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13735	switch (uIntType)
13736	{
13737	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13738	Assert(uVector == X86_XCPT_DB);
13739	RT_FALL_THRU();
13740	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13741	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13742	RT_FALL_THRU();
13743	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13744	{
13745	/*
13746	* If there's any exception caused as a result of event injection, the resulting
13747	* secondary/final execption will be pending, we shall continue guest execution
13748	* after injecting the event. The page-fault case is complicated and we manually
13749	* handle any currently pending event in hmR0VmxExitXcptPF.
13750	*/
13751	if (!pVCpu->hm.s.Event.fPending)
13752	{ /* likely */ }
13753	else if (uVector != X86_XCPT_PF)
13754	{
13755	rcStrict = VINF_SUCCESS;
13756	break;
13757	}
13758
13759	switch (uVector)
13760	{
13761	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13762	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13763	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13764	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13765	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13766	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13767
13768	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13769	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13770	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13771	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13772	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13773	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13774	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13775	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13776	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13777	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13778	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13779	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13780	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13781	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13782	default:
13783	{
13784	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13785	if (pVmcsInfo->RealMode.fRealOnV86Active)
13786	{
13787	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13788	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13789	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13790
13791	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13792	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13793	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13794	AssertRCReturn(rc, rc);
13795	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13796	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13797	0 /* GCPtrFaultAddress */);
13798	rcStrict = VINF_SUCCESS;
13799	}
13800	else
13801	{
13802	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13803	pVCpu->hm.s.u32HMError = uVector;
13804	rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
13805	}
13806	break;
13807	}
13808	}
13809	break;
13810	}
13811
13812	default:
13813	{
13814	pVCpu->hm.s.u32HMError = uExitIntInfo;
13815	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13816	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13817	break;
13818	}
13819	}
13820	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13821	return rcStrict;
13822	}
13823
13824
13825	/**
13826	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13827	*/
13828	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13829	{
13830	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13831
13832	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13833	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13834	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13835	AssertRCReturn(rc, rc);
13836
13837	/* Evaluate and deliver pending events and resume guest execution. */
13838	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13839	return VINF_SUCCESS;
13840	}
13841
13842
13843	/**
13844	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13845	*/
13846	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13847	{
13848	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13849
13850	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13851	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13852	{
13853	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13854	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13855	}
13856
13857	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
13858
13859	/*
13860	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13861	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13862	*/
13863	uint32_t fIntrState;
13864	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13865	AssertRCReturn(rc, rc);
13866	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13867	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13868	{
13869	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13870	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13871
13872	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13873	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13874	AssertRCReturn(rc, rc);
13875	}
13876
13877	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13878	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13879	AssertRCReturn(rc, rc);
13880
13881	/* Evaluate and deliver pending events and resume guest execution. */
13882	return VINF_SUCCESS;
13883	}
13884
13885
13886	/**
13887	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13888	*/
13889	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13890	{
13891	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13892	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13893	}
13894
13895
13896	/**
13897	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13898	*/
13899	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13900	{
13901	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13902	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13903	}
13904
13905
13906	/**
13907	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13908	*/
13909	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13910	{
13911	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13912
13913	/*
13914	* Get the state we need and update the exit history entry.
13915	*/
13916	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13917	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13918	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13919	AssertRCReturn(rc, rc);
13920
13921	VBOXSTRICTRC rcStrict;
13922	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13923	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13924	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13925	if (!pExitRec)
13926	{
13927	/*
13928	* Regular CPUID instruction execution.
13929	*/
13930	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13931	if (rcStrict == VINF_SUCCESS)
13932	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13933	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13934	{
13935	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13936	rcStrict = VINF_SUCCESS;
13937	}
13938	}
13939	else
13940	{
13941	/*
13942	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13943	*/
13944	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13945	AssertRCReturn(rc2, rc2);
13946
13947	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13948	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13949
13950	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13951	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13952
13953	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13954	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13955	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13956	}
13957	return rcStrict;
13958	}
13959
13960
13961	/**
13962	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13963	*/
13964	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13965	{
13966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13967
13968	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13969	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13970	AssertRCReturn(rc, rc);
13971
13972	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13973	return VINF_EM_RAW_EMULATE_INSTR;
13974
13975	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
13976	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13977	}
13978
13979
13980	/**
13981	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13982	*/
13983	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13984	{
13985	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13986
13987	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13988	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13989	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13990	AssertRCReturn(rc, rc);
13991
13992	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13993	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13994	{
13995	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13996	we must reset offsetting on VM-entry. See @bugref{6634}. */
13997	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13998	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13999	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14000	}
14001	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14002	{
14003	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14004	rcStrict = VINF_SUCCESS;
14005	}
14006	return rcStrict;
14007	}
14008
14009
14010	/**
14011	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14012	*/
14013	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14014	{
14015	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14016
14017	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14018	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14019	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14020	AssertRCReturn(rc, rc);
14021
14022	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
14023	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14024	{
14025	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14026	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14027	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14028	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14029	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14030	}
14031	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14032	{
14033	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14034	rcStrict = VINF_SUCCESS;
14035	}
14036	return rcStrict;
14037	}
14038
14039
14040	/**
14041	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14042	*/
14043	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14044	{
14045	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14046
14047	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14048	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14049	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14050	AssertRCReturn(rc, rc);
14051
14052	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14053	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14054	if (RT_LIKELY(rc == VINF_SUCCESS))
14055	{
14056	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14057	Assert(pVmxTransient->cbInstr == 2);
14058	}
14059	else
14060	{
14061	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14062	rc = VERR_EM_INTERPRETER;
14063	}
14064	return rc;
14065	}
14066
14067
14068	/**
14069	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14070	*/
14071	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14072	{
14073	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14074
14075	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14076	if (EMAreHypercallInstructionsEnabled(pVCpu))
14077	{
14078	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14079	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14080	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14081	AssertRCReturn(rc, rc);
14082
14083	/* Perform the hypercall. */
14084	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14085	if (rcStrict == VINF_SUCCESS)
14086	{
14087	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14088	AssertRCReturn(rc, rc);
14089	}
14090	else
14091	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14092	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14093	\|\| RT_FAILURE(rcStrict));
14094
14095	/* If the hypercall changes anything other than guest's general-purpose registers,
14096	we would need to reload the guest changed bits here before VM-entry. */
14097	}
14098	else
14099	Log4Func(("Hypercalls not enabled\n"));
14100
14101	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14102	if (RT_FAILURE(rcStrict))
14103	{
14104	hmR0VmxSetPendingXcptUD(pVCpu);
14105	rcStrict = VINF_SUCCESS;
14106	}
14107
14108	return rcStrict;
14109	}
14110
14111
14112	/**
14113	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14114	*/
14115	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14116	{
14117	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14118	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14119
14120	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14121	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14122	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14123	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14124	AssertRCReturn(rc, rc);
14125
14126	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
14127
14128	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14129	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14130	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14131	{
14132	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14133	rcStrict = VINF_SUCCESS;
14134	}
14135	else
14136	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14137	VBOXSTRICTRC_VAL(rcStrict)));
14138	return rcStrict;
14139	}
14140
14141
14142	/**
14143	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14144	*/
14145	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14146	{
14147	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14148
14149	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14150	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14151	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14152	AssertRCReturn(rc, rc);
14153
14154	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
14155	if (rcStrict == VINF_SUCCESS)
14156	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14157	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14158	{
14159	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14160	rcStrict = VINF_SUCCESS;
14161	}
14162
14163	return rcStrict;
14164	}
14165
14166
14167	/**
14168	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14169	*/
14170	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14171	{
14172	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14173
14174	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14175	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14176	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14177	AssertRCReturn(rc, rc);
14178
14179	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
14180	if (RT_SUCCESS(rcStrict))
14181	{
14182	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14183	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14184	rcStrict = VINF_SUCCESS;
14185	}
14186
14187	return rcStrict;
14188	}
14189
14190
14191	/**
14192	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14193	* VM-exit.
14194	*/
14195	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14196	{
14197	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14198	return VINF_EM_RESET;
14199	}
14200
14201
14202	/**
14203	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14204	*/
14205	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14206	{
14207	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14208
14209	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14210	AssertRCReturn(rc, rc);
14211
14212	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14213	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14214	rc = VINF_SUCCESS;
14215	else
14216	rc = VINF_EM_HALT;
14217
14218	if (rc != VINF_SUCCESS)
14219	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14220	return rc;
14221	}
14222
14223
14224	/**
14225	* VM-exit handler for instructions that result in a \#UD exception delivered to
14226	* the guest.
14227	*/
14228	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14229	{
14230	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14231	hmR0VmxSetPendingXcptUD(pVCpu);
14232	return VINF_SUCCESS;
14233	}
14234
14235
14236	/**
14237	* VM-exit handler for expiry of the VMX-preemption timer.
14238	*/
14239	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14240	{
14241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14242
14243	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14244	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14245
14246	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14247	PVM pVM = pVCpu->CTX_SUFF(pVM);
14248	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14249	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14250	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14251	}
14252
14253
14254	/**
14255	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14256	*/
14257	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14258	{
14259	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14260
14261	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14262	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14263	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14264	AssertRCReturn(rc, rc);
14265
14266	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
14267	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14268	: HM_CHANGED_RAISED_XCPT_MASK);
14269
14270	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14271	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14272
14273	return rcStrict;
14274	}
14275
14276
14277	/**
14278	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14279	*/
14280	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14281	{
14282	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14283	/** @todo Use VM-exit instruction information. */
14284	return VERR_EM_INTERPRETER;
14285	}
14286
14287
14288	/**
14289	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14290	* VM-exit.
14291	*/
14292	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14293	{
14294	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14295	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14296	AssertRCReturn(rc, rc);
14297
14298	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
14299	if (RT_FAILURE(rc))
14300	return rc;
14301
14302	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14303	NOREF(uInvalidReason);
14304
14305	#ifdef VBOX_STRICT
14306	uint32_t fIntrState;
14307	RTHCUINTREG uHCReg;
14308	uint64_t u64Val;
14309	uint32_t u32Val;
14310	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14311	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14312	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14313	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14314	AssertRCReturn(rc, rc);
14315
14316	Log4(("uInvalidReason %u\n", uInvalidReason));
14317	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14318	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14319	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14320	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14321
14322	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
14323	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
14324	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
14325	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
14326	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
14327	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14328	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
14329	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
14330	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
14331	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14332	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14333	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14334
14335	hmR0DumpRegs(pVCpu);
14336	#endif
14337
14338	return VERR_VMX_INVALID_GUEST_STATE;
14339	}
14340
14341	/**
14342	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14343	*/
14344	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14345	{
14346	/*
14347	* Cummulative notes of all recognized but unexpected VM-exits.
14348	*
14349	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
14350	* nested-paging is used.
14351	*
14352	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14353	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14354	* this function (and thereby stop VM execution) for handling such instructions.
14355	*
14356	*
14357	* VMX_EXIT_INIT_SIGNAL:
14358	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14359	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14360	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14361	*
14362	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14363	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14364	* See Intel spec. "23.8 Restrictions on VMX operation".
14365	*
14366	* VMX_EXIT_SIPI:
14367	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14368	* activity state is used. We don't make use of it as our guests don't have direct
14369	* access to the host local APIC.
14370	*
14371	* See Intel spec. 25.3 "Other Causes of VM-exits".
14372	*
14373	* VMX_EXIT_IO_SMI:
14374	* VMX_EXIT_SMI:
14375	* This can only happen if we support dual-monitor treatment of SMI, which can be
14376	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14377	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14378	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14379	*
14380	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14381	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14382	*
14383	* VMX_EXIT_ERR_MSR_LOAD:
14384	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14385	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14386	* execution.
14387	*
14388	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14389	*
14390	* VMX_EXIT_ERR_MACHINE_CHECK:
14391	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14392	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14393	* #MC exception abort class exception is raised. We thus cannot assume a
14394	* reasonable chance of continuing any sort of execution and we bail.
14395	*
14396	* See Intel spec. 15.1 "Machine-check Architecture".
14397	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14398	*
14399	* VMX_EXIT_PML_FULL:
14400	* VMX_EXIT_VIRTUALIZED_EOI:
14401	* VMX_EXIT_APIC_WRITE:
14402	* We do not currently support any of these features and thus they are all unexpected
14403	* VM-exits.
14404	*
14405	* VMX_EXIT_GDTR_IDTR_ACCESS:
14406	* VMX_EXIT_LDTR_TR_ACCESS:
14407	* VMX_EXIT_RDRAND:
14408	* VMX_EXIT_RSM:
14409	* VMX_EXIT_VMFUNC:
14410	* VMX_EXIT_ENCLS:
14411	* VMX_EXIT_RDSEED:
14412	* VMX_EXIT_XSAVES:
14413	* VMX_EXIT_XRSTORS:
14414	* VMX_EXIT_UMWAIT:
14415	* VMX_EXIT_TPAUSE:
14416	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14417	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14418	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14419	*
14420	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14421	*/
14422	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14423	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14424	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14425	}
14426
14427
14428	/**
14429	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14430	*/
14431	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14432	{
14433	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14434
14435	/** @todo Optimize this: We currently drag in in the whole MSR state
14436	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14437	* MSRs required. That would require changes to IEM and possibly CPUM too.
14438	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14439	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14440	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14441	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14442	switch (idMsr)
14443	{
14444	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14445	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14446	}
14447
14448	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14449	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14450	AssertRCReturn(rc, rc);
14451
14452	Log4Func(("ecx=%#RX32\n", idMsr));
14453
14454	#ifdef VBOX_STRICT
14455	if (hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14456	{
14457	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14458	&& idMsr != MSR_K6_EFER)
14459	{
14460	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14461	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14462	}
14463	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14464	{
14465	Assert(pVmcsInfo->pvMsrBitmap);
14466	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14467	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14468	{
14469	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14470	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14471	}
14472	}
14473	}
14474	#endif
14475
14476	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14477	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14478	if (rcStrict == VINF_SUCCESS)
14479	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14480	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14481	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14482	{
14483	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14484	rcStrict = VINF_SUCCESS;
14485	}
14486	else
14487	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14488
14489	return rcStrict;
14490	}
14491
14492
14493	/**
14494	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14495	*/
14496	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14497	{
14498	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14499
14500	/** @todo Optimize this: We currently drag in in the whole MSR state
14501	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14502	* MSRs required. That would require changes to IEM and possibly CPUM too.
14503	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14504	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14505	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14506
14507	/*
14508	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14509	* Although we don't need to fetch the base as it will be overwritten shortly, while
14510	* loading guest-state we would also load the entire segment register including limit
14511	* and attributes and thus we need to load them here.
14512	*/
14513	switch (idMsr)
14514	{
14515	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14516	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14517	}
14518
14519	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14520	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14521	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14522	AssertRCReturn(rc, rc);
14523
14524	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14525
14526	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14528
14529	if (rcStrict == VINF_SUCCESS)
14530	{
14531	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14532
14533	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14534	if ( idMsr == MSR_IA32_APICBASE
14535	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14536	&& idMsr <= MSR_IA32_X2APIC_END))
14537	{
14538	/*
14539	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14540	* When full APIC register virtualization is implemented we'll have to make
14541	* sure APIC state is saved from the VMCS before IEM changes it.
14542	*/
14543	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14544	}
14545	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14546	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14547	else if (idMsr == MSR_K6_EFER)
14548	{
14549	/*
14550	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14551	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14552	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14553	*/
14554	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14555	}
14556
14557	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14558	if (!hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14559	{
14560	switch (idMsr)
14561	{
14562	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14563	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14564	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14565	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14566	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14567	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14568	default:
14569	{
14570	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14571	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14572	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14573	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14574	break;
14575	}
14576	}
14577	}
14578	#ifdef VBOX_STRICT
14579	else
14580	{
14581	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14582	switch (idMsr)
14583	{
14584	case MSR_IA32_SYSENTER_CS:
14585	case MSR_IA32_SYSENTER_EIP:
14586	case MSR_IA32_SYSENTER_ESP:
14587	case MSR_K8_FS_BASE:
14588	case MSR_K8_GS_BASE:
14589	{
14590	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14591	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14592	}
14593
14594	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14595	default:
14596	{
14597	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14598	{
14599	/* EFER MSR writes are always intercepted. */
14600	if (idMsr != MSR_K6_EFER)
14601	{
14602	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14603	idMsr));
14604	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14605	}
14606	}
14607
14608	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14609	{
14610	Assert(pVmcsInfo->pvMsrBitmap);
14611	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14612	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14613	{
14614	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14615	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14616	}
14617	}
14618	break;
14619	}
14620	}
14621	}
14622	#endif /* VBOX_STRICT */
14623	}
14624	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14625	{
14626	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14627	rcStrict = VINF_SUCCESS;
14628	}
14629	else
14630	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14631
14632	return rcStrict;
14633	}
14634
14635
14636	/**
14637	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14638	*/
14639	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14640	{
14641	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14642
14643	/** @todo The guest has likely hit a contended spinlock. We might want to
14644	* poke a schedule different guest VCPU. */
14645	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14646	if (RT_SUCCESS(rc))
14647	return VINF_EM_RAW_INTERRUPT;
14648
14649	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14650	return rc;
14651	}
14652
14653
14654	/**
14655	* VM-exit handler for when the TPR value is lowered below the specified
14656	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14657	*/
14658	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14659	{
14660	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14661	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14662
14663	/*
14664	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14665	* We'll re-evaluate pending interrupts and inject them before the next VM
14666	* entry so we can just continue execution here.
14667	*/
14668	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14669	return VINF_SUCCESS;
14670	}
14671
14672
14673	/**
14674	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14675	* VM-exit.
14676	*
14677	* @retval VINF_SUCCESS when guest execution can continue.
14678	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14679	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
14680	* incompatible guest state for VMX execution (real-on-v86 case).
14681	*/
14682	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14683	{
14684	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14685	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14686
14687	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14688	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14689	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14690	AssertRCReturn(rc, rc);
14691
14692	VBOXSTRICTRC rcStrict;
14693	PVM pVM = pVCpu->CTX_SUFF(pVM);
14694	uint64_t const uExitQual = pVmxTransient->uExitQual;
14695	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14696	switch (uAccessType)
14697	{
14698	/*
14699	* MOV to CRx.
14700	*/
14701	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
14702	{
14703	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14704	AssertRCReturn(rc, rc);
14705
14706	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
14707	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14708	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14709	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14710
14711	/*
14712	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
14713	* - When nested paging isn't used.
14714	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
14715	* - We are executing in the VM debug loop.
14716	*/
14717	Assert( iCrReg != 3
14718	\|\| !pVM->hm.s.fNestedPaging
14719	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14720	\|\| pVCpu->hm.s.fUsingDebugLoop);
14721
14722	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
14723	Assert( iCrReg != 8
14724	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14725
14726	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14727	AssertMsg( rcStrict == VINF_SUCCESS
14728	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14729
14730	/*
14731	* This is a kludge for handling switches back to real mode when we try to use
14732	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14733	* deal with special selector values, so we have to return to ring-3 and run
14734	* there till the selector values are V86 mode compatible.
14735	*
14736	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14737	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
14738	* this function.
14739	*/
14740	if ( iCrReg == 0
14741	&& rcStrict == VINF_SUCCESS
14742	&& !pVM->hm.s.vmx.fUnrestrictedGuest
14743	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14744	&& (uOldCr0 & X86_CR0_PE)
14745	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
14746	{
14747	/** @todo Check selectors rather than returning all the time. */
14748	Assert(!pVmxTransient->fIsNestedGuest);
14749	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14750	rcStrict = VINF_EM_RESCHEDULE_REM;
14751	}
14752	break;
14753	}
14754
14755	/*
14756	* MOV from CRx.
14757	*/
14758	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
14759	{
14760	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14761	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14762
14763	/*
14764	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
14765	* - When nested paging isn't used.
14766	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
14767	* - We are executing in the VM debug loop.
14768	*/
14769	Assert( iCrReg != 3
14770	\|\| !pVM->hm.s.fNestedPaging
14771	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14772	\|\| pVCpu->hm.s.fUsingDebugLoop);
14773
14774	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14775	Assert( iCrReg != 8
14776	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14777
14778	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14779	break;
14780	}
14781
14782	/*
14783	* CLTS (Clear Task-Switch Flag in CR0).
14784	*/
14785	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
14786	{
14787	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
14788	break;
14789	}
14790
14791	/*
14792	* LMSW (Load Machine-Status Word into CR0).
14793	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
14794	*/
14795	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
14796	{
14797	RTGCPTR GCPtrEffDst;
14798	uint8_t const cbInstr = pVmxTransient->cbInstr;
14799	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
14800	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
14801	if (fMemOperand)
14802	{
14803	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14804	AssertRCReturn(rc, rc);
14805	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
14806	}
14807	else
14808	GCPtrEffDst = NIL_RTGCPTR;
14809	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
14810	break;
14811	}
14812
14813	default:
14814	{
14815	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
14816	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
14817	}
14818	}
14819
14820	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14821	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14822	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
14823
14824	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14825	NOREF(pVM);
14826	return rcStrict;
14827	}
14828
14829
14830	/**
14831	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14832	* VM-exit.
14833	*/
14834	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14835	{
14836	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14837	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14838
14839	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14840	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14841	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14842	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14843	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14844	\| CPUMCTX_EXTRN_EFER);
14845	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14846	AssertRCReturn(rc, rc);
14847
14848	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14849	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14850	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14851	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14852	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14853	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14854	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14855	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14856
14857	/*
14858	* Update exit history to see if this exit can be optimized.
14859	*/
14860	VBOXSTRICTRC rcStrict;
14861	PCEMEXITREC pExitRec = NULL;
14862	if ( !fGstStepping
14863	&& !fDbgStepping)
14864	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14865	!fIOString
14866	? !fIOWrite
14867	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14868	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14869	: !fIOWrite
14870	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14871	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14872	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14873	if (!pExitRec)
14874	{
14875	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14876	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14877
14878	uint32_t const cbValue = s_aIOSizes[uIOSize];
14879	uint32_t const cbInstr = pVmxTransient->cbInstr;
14880	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14881	PVM pVM = pVCpu->CTX_SUFF(pVM);
14882	if (fIOString)
14883	{
14884	/*
14885	* INS/OUTS - I/O String instruction.
14886	*
14887	* Use instruction-information if available, otherwise fall back on
14888	* interpreting the instruction.
14889	*/
14890	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14891	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14892	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14893	if (fInsOutsInfo)
14894	{
14895	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14896	AssertRCReturn(rc2, rc2);
14897	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14898	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14899	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14900	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14901	if (fIOWrite)
14902	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14903	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14904	else
14905	{
14906	/*
14907	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14908	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14909	* See Intel Instruction spec. for "INS".
14910	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14911	*/
14912	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14913	}
14914	}
14915	else
14916	rcStrict = IEMExecOne(pVCpu);
14917
14918	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14919	fUpdateRipAlready = true;
14920	}
14921	else
14922	{
14923	/*
14924	* IN/OUT - I/O instruction.
14925	*/
14926	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14927	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14928	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14929	if (fIOWrite)
14930	{
14931	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14933	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14934	&& !pCtx->eflags.Bits.u1TF)
14935	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14936	}
14937	else
14938	{
14939	uint32_t u32Result = 0;
14940	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14941	if (IOM_SUCCESS(rcStrict))
14942	{
14943	/* Save result of I/O IN instr. in AL/AX/EAX. */
14944	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14945	}
14946	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14947	&& !pCtx->eflags.Bits.u1TF)
14948	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14950	}
14951	}
14952
14953	if (IOM_SUCCESS(rcStrict))
14954	{
14955	if (!fUpdateRipAlready)
14956	{
14957	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14958	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14959	}
14960
14961	/*
14962	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14963	* while booting Fedora 17 64-bit guest.
14964	*
14965	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14966	*/
14967	if (fIOString)
14968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14969
14970	/*
14971	* If any I/O breakpoints are armed, we need to check if one triggered
14972	* and take appropriate action.
14973	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14974	*/
14975	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14976	AssertRCReturn(rc, rc);
14977
14978	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14979	* execution engines about whether hyper BPs and such are pending. */
14980	uint32_t const uDr7 = pCtx->dr[7];
14981	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14982	&& X86_DR7_ANY_RW_IO(uDr7)
14983	&& (pCtx->cr4 & X86_CR4_DE))
14984	\|\| DBGFBpIsHwIoArmed(pVM)))
14985	{
14986	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14987
14988	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14989	VMMRZCallRing3Disable(pVCpu);
14990	HM_DISABLE_PREEMPT(pVCpu);
14991
14992	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14993
14994	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14995	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14996	{
14997	/* Raise #DB. */
14998	if (fIsGuestDbgActive)
14999	ASMSetDR6(pCtx->dr[6]);
15000	if (pCtx->dr[7] != uDr7)
15001	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15002
15003	hmR0VmxSetPendingXcptDB(pVCpu);
15004	}
15005	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15006	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15007	else if ( rcStrict2 != VINF_SUCCESS
15008	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15009	rcStrict = rcStrict2;
15010	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15011
15012	HM_RESTORE_PREEMPT();
15013	VMMRZCallRing3Enable(pVCpu);
15014	}
15015	}
15016
15017	#ifdef VBOX_STRICT
15018	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15019	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15020	Assert(!fIOWrite);
15021	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15022	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15023	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15024	Assert(fIOWrite);
15025	else
15026	{
15027	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15028	* statuses, that the VMM device and some others may return. See
15029	* IOM_SUCCESS() for guidance. */
15030	AssertMsg( RT_FAILURE(rcStrict)
15031	\|\| rcStrict == VINF_SUCCESS
15032	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15033	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15034	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15035	\|\| rcStrict == VINF_EM_RAW_TO_R3
15036	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15037	# endif
15038	}
15039	#endif
15040	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15041	}
15042	else
15043	{
15044	/*
15045	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15046	*/
15047	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15048	AssertRCReturn(rc2, rc2);
15049	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15050	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15051	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15052	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15053	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15054	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15055
15056	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15057	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15058
15059	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15060	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15061	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15062	}
15063	return rcStrict;
15064	}
15065
15066
15067	/**
15068	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15069	* VM-exit.
15070	*/
15071	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15072	{
15073	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15074
15075	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15076	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15077	AssertRCReturn(rc, rc);
15078	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15079	{
15080	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15081	AssertRCReturn(rc, rc);
15082	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15083	{
15084	uint32_t uErrCode;
15085	RTGCUINTPTR GCPtrFaultAddress;
15086	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
15087	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
15088	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
15089	if (fErrorCodeValid)
15090	{
15091	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15092	AssertRCReturn(rc, rc);
15093	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15094	}
15095	else
15096	uErrCode = 0;
15097
15098	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
15099	&& uVector == X86_XCPT_PF)
15100	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15101	else
15102	GCPtrFaultAddress = 0;
15103
15104	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15105	AssertRCReturn(rc, rc);
15106
15107	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15108	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
15109
15110	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
15111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15112	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15113	}
15114	}
15115
15116	/* Fall back to the interpreter to emulate the task-switch. */
15117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15118	return VERR_EM_INTERPRETER;
15119	}
15120
15121
15122	/**
15123	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15124	*/
15125	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15126	{
15127	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15128
15129	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15130	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15131	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15132	AssertRCReturn(rc, rc);
15133	return VINF_EM_DBG_STEPPED;
15134	}
15135
15136
15137	/**
15138	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15139	*/
15140	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15141	{
15142	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15143	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15144
15145	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15146	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15147	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15148	{
15149	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15150	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15151	{
15152	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
15153	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15154	}
15155	}
15156	else
15157	{
15158	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15159	rcStrict1 = VINF_SUCCESS;
15160	return rcStrict1;
15161	}
15162
15163	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15164	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15165	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15166	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15167	AssertRCReturn(rc, rc);
15168
15169	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15170	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15171	VBOXSTRICTRC rcStrict2;
15172	switch (uAccessType)
15173	{
15174	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15175	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15176	{
15177	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15178	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15179	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15180
15181	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15182	GCPhys &= PAGE_BASE_GC_MASK;
15183	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15184	PVM pVM = pVCpu->CTX_SUFF(pVM);
15185	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15186	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15187
15188	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15189	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
15190	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15191	CPUMCTX2CORE(pCtx), GCPhys);
15192	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15193	if ( rcStrict2 == VINF_SUCCESS
15194	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15195	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
15196	{
15197	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15198	\| HM_CHANGED_GUEST_APIC_TPR);
15199	rcStrict2 = VINF_SUCCESS;
15200	}
15201	break;
15202	}
15203
15204	default:
15205	Log4Func(("uAccessType=%#x\n", uAccessType));
15206	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
15207	break;
15208	}
15209
15210	if (rcStrict2 != VINF_SUCCESS)
15211	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15212	return rcStrict2;
15213	}
15214
15215
15216	/**
15217	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15218	* VM-exit.
15219	*/
15220	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15221	{
15222	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15223
15224	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15225	if (pVmxTransient->fWasGuestDebugStateActive)
15226	{
15227	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15228	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15229	}
15230
15231	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15232	if ( !pVCpu->hm.s.fSingleInstruction
15233	&& !pVmxTransient->fWasHyperDebugStateActive)
15234	{
15235	Assert(!DBGFIsStepping(pVCpu));
15236	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15237
15238	/* Don't intercept MOV DRx any more. */
15239	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15240	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15241	AssertRCReturn(rc, rc);
15242
15243	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15244	VMMRZCallRing3Disable(pVCpu);
15245	HM_DISABLE_PREEMPT(pVCpu);
15246
15247	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15248	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15249	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
15250
15251	HM_RESTORE_PREEMPT();
15252	VMMRZCallRing3Enable(pVCpu);
15253
15254	#ifdef VBOX_WITH_STATISTICS
15255	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15256	AssertRCReturn(rc, rc);
15257	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15258	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15259	else
15260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15261	#endif
15262	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15263	return VINF_SUCCESS;
15264	}
15265
15266	/*
15267	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15268	* The EFER MSR is always up-to-date.
15269	* Update the segment registers and DR7 from the CPU.
15270	*/
15271	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15272	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15273	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15274	AssertRCReturn(rc, rc);
15275	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15276
15277	PVM pVM = pVCpu->CTX_SUFF(pVM);
15278	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15279	{
15280	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15281	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15282	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15283	if (RT_SUCCESS(rc))
15284	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15286	}
15287	else
15288	{
15289	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15290	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15291	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15292	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15293	}
15294
15295	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15296	if (RT_SUCCESS(rc))
15297	{
15298	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15299	AssertRCReturn(rc2, rc2);
15300	return VINF_SUCCESS;
15301	}
15302	return rc;
15303	}
15304
15305
15306	/**
15307	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15308	* Conditional VM-exit.
15309	*/
15310	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15311	{
15312	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15313	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15314
15315	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15316	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15317	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15318	{
15319	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
15320	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
15321	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15322	{
15323	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
15324	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15325	}
15326	}
15327	else
15328	{
15329	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15330	rcStrict1 = VINF_SUCCESS;
15331	return rcStrict1;
15332	}
15333
15334	/*
15335	* Get sufficent state and update the exit history entry.
15336	*/
15337	RTGCPHYS GCPhys;
15338	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15339	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15340	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15341	AssertRCReturn(rc, rc);
15342
15343	VBOXSTRICTRC rcStrict;
15344	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15345	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15346	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15347	if (!pExitRec)
15348	{
15349	/*
15350	* If we succeed, resume guest execution.
15351	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15352	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15353	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15354	* weird case. See @bugref{6043}.
15355	*/
15356	PVM pVM = pVCpu->CTX_SUFF(pVM);
15357	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15358	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15359	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15360	if ( rcStrict == VINF_SUCCESS
15361	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15362	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15363	{
15364	/* Successfully handled MMIO operation. */
15365	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15366	\| HM_CHANGED_GUEST_APIC_TPR);
15367	rcStrict = VINF_SUCCESS;
15368	}
15369	}
15370	else
15371	{
15372	/*
15373	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15374	*/
15375	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15376	AssertRCReturn(rc2, rc2);
15377
15378	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15379	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15380
15381	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15382	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15383
15384	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15385	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15386	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15387	}
15388	return VBOXSTRICTRC_TODO(rcStrict);
15389	}
15390
15391
15392	/**
15393	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15394	* VM-exit.
15395	*/
15396	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15397	{
15398	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15399	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15400
15401	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15402	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15403	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15404	{
15405	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
15406	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15407	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
15408	}
15409	else
15410	{
15411	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15412	rcStrict1 = VINF_SUCCESS;
15413	return rcStrict1;
15414	}
15415
15416	RTGCPHYS GCPhys;
15417	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15418	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15419	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15420	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15421	AssertRCReturn(rc, rc);
15422
15423	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
15424	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
15425
15426	RTGCUINT uErrorCode = 0;
15427	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15428	uErrorCode \|= X86_TRAP_PF_ID;
15429	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15430	uErrorCode \|= X86_TRAP_PF_RW;
15431	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15432	uErrorCode \|= X86_TRAP_PF_P;
15433
15434	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15435
15436
15437	/* Handle the pagefault trap for the nested shadow table. */
15438	PVM pVM = pVCpu->CTX_SUFF(pVM);
15439	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15440
15441	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
15442	pCtx->cs.Sel, pCtx->rip));
15443
15444	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15445	TRPMResetTrap(pVCpu);
15446
15447	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15448	if ( rcStrict2 == VINF_SUCCESS
15449	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15450	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
15451	{
15452	/* Successfully synced our nested page tables. */
15453	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15454	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15455	return VINF_SUCCESS;
15456	}
15457
15458	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15459	return rcStrict2;
15460	}
15461
15462	/** @} */
15463
15464	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15465	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
15466	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15467
15468	/**
15469	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
15470	*/
15471	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15472	{
15473	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15474	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15475
15476	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15477	AssertRCReturn(rc, rc);
15478
15479	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15480	{
15481	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15482	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15483
15484	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15485	* provides VM-exit instruction length. If this causes problem later,
15486	* disassemble the instruction like it's done on AMD-V. */
15487	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15488	AssertRCReturn(rc2, rc2);
15489	return rc;
15490	}
15491
15492	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15493	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15494	return rc;
15495	}
15496
15497
15498	/**
15499	* VM-exit exception handler for \#BP (Breakpoint exception).
15500	*/
15501	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15502	{
15503	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15505
15506	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15507	AssertRCReturn(rc, rc);
15508
15509	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15510	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15511	if (rc == VINF_EM_RAW_GUEST_TRAP)
15512	{
15513	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15514	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15515	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15516	AssertRCReturn(rc, rc);
15517
15518	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15519	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15520	}
15521
15522	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15523	return rc;
15524	}
15525
15526
15527	/**
15528	* VM-exit exception handler for \#AC (alignment check exception).
15529	*/
15530	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15531	{
15532	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15533
15534	/*
15535	* Re-inject it. We'll detect any nesting before getting here.
15536	*/
15537	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15538	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15539	AssertRCReturn(rc, rc);
15540	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15541
15542	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15543	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15544	return VINF_SUCCESS;
15545	}
15546
15547
15548	/**
15549	* VM-exit exception handler for \#DB (Debug exception).
15550	*/
15551	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15552	{
15553	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15554	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15555
15556	/*
15557	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
15558	*/
15559	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15560
15561	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15562	uint64_t const uDR6 = X86_DR6_INIT_VAL
15563	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15564	\| X86_DR6_BD \| X86_DR6_BS));
15565
15566	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15567	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15568	Log6Func(("rc=%Rrc\n", rc));
15569	if (rc == VINF_EM_RAW_GUEST_TRAP)
15570	{
15571	/*
15572	* The exception was for the guest. Update DR6, DR7.GD and
15573	* IA32_DEBUGCTL.LBR before forwarding it.
15574	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15575	*/
15576	VMMRZCallRing3Disable(pVCpu);
15577	HM_DISABLE_PREEMPT(pVCpu);
15578
15579	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15580	pCtx->dr[6] \|= uDR6;
15581	if (CPUMIsGuestDebugStateActive(pVCpu))
15582	ASMSetDR6(pCtx->dr[6]);
15583
15584	HM_RESTORE_PREEMPT();
15585	VMMRZCallRing3Enable(pVCpu);
15586
15587	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15588	AssertRCReturn(rc, rc);
15589
15590	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15591	pCtx->dr[7] &= ~X86_DR7_GD;
15592
15593	/* Paranoia. */
15594	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15595	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15596
15597	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15598	AssertRCReturn(rc, rc);
15599
15600	/*
15601	* Raise #DB in the guest.
15602	*
15603	* It is important to reflect exactly what the VM-exit gave us (preserving the
15604	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15605	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15606	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15607	*
15608	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15609	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15610	*/
15611	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15612	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15613	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15614	AssertRCReturn(rc, rc);
15615	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15616	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15617	return VINF_SUCCESS;
15618	}
15619
15620	/*
15621	* Not a guest trap, must be a hypervisor related debug event then.
15622	* Update DR6 in case someone is interested in it.
15623	*/
15624	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15625	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15626	CPUMSetHyperDR6(pVCpu, uDR6);
15627
15628	return rc;
15629	}
15630
15631
15632	/**
15633	* Hacks its way around the lovely mesa driver's backdoor accesses.
15634	*
15635	* @sa hmR0SvmHandleMesaDrvGp.
15636	*/
15637	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15638	{
15639	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15640	RT_NOREF(pCtx);
15641
15642	/* For now we'll just skip the instruction. */
15643	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15644	}
15645
15646
15647	/**
15648	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15649	* backdoor logging w/o checking what it is running inside.
15650	*
15651	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15652	* backdoor port and magic numbers loaded in registers.
15653	*
15654	* @returns true if it is, false if it isn't.
15655	* @sa hmR0SvmIsMesaDrvGp.
15656	*/
15657	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15658	{
15659	/* 0xed: IN eAX,dx */
15660	uint8_t abInstr[1];
15661	if (pVmxTransient->cbInstr != sizeof(abInstr))
15662	return false;
15663
15664	/* Check that it is #GP(0). */
15665	if (pVmxTransient->uExitIntErrorCode != 0)
15666	return false;
15667
15668	/* Check magic and port. */
15669	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15670	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15671	if (pCtx->rax != UINT32_C(0x564d5868))
15672	return false;
15673	if (pCtx->dx != UINT32_C(0x5658))
15674	return false;
15675
15676	/* Flat ring-3 CS. */
15677	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15678	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15679	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15680	if (pCtx->cs.Attr.n.u2Dpl != 3)
15681	return false;
15682	if (pCtx->cs.u64Base != 0)
15683	return false;
15684
15685	/* Check opcode. */
15686	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15687	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15688	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15689	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15690	if (RT_FAILURE(rc))
15691	return false;
15692	if (abInstr[0] != 0xed)
15693	return false;
15694
15695	return true;
15696	}
15697
15698
15699	/**
15700	* VM-exit exception handler for \#GP (General-protection exception).
15701	*
15702	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15703	*/
15704	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15705	{
15706	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15708
15709	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15710	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15711	if (pVmcsInfo->RealMode.fRealOnV86Active)
15712	{ /* likely */ }
15713	else
15714	{
15715	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15716	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
15717	#endif
15718	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15719	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15720	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15721	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15722	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15723	AssertRCReturn(rc, rc);
15724	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15725	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15726
15727	if ( pVmxTransient->fIsNestedGuest
15728	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15729	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15730	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15731	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15732	else
15733	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15734	return rc;
15735	}
15736
15737	Assert(CPUMIsGuestInRealModeEx(pCtx));
15738	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15739	Assert(!pVmxTransient->fIsNestedGuest);
15740
15741	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15742	AssertRCReturn(rc, rc);
15743
15744	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15745	if (rcStrict == VINF_SUCCESS)
15746	{
15747	if (!CPUMIsGuestInRealModeEx(pCtx))
15748	{
15749	/*
15750	* The guest is no longer in real-mode, check if we can continue executing the
15751	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15752	*/
15753	pVmcsInfo->RealMode.fRealOnV86Active = false;
15754	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15755	{
15756	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15757	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15758	}
15759	else
15760	{
15761	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15762	rcStrict = VINF_EM_RESCHEDULE;
15763	}
15764	}
15765	else
15766	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15767	}
15768	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15769	{
15770	rcStrict = VINF_SUCCESS;
15771	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15772	}
15773	return VBOXSTRICTRC_VAL(rcStrict);
15774	}
15775
15776
15777	/**
15778	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15779	* the exception reported in the VMX transient structure back into the VM.
15780	*
15781	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15782	* up-to-date.
15783	*/
15784	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15785	{
15786	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15787	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15788	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15789	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
15790	("uVector=%#x u32XcptBitmap=%#X32\n",
15791	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15792	NOREF(pVmcsInfo);
15793	#endif
15794
15795	/*
15796	* Re-inject the exception into the guest. This cannot be a double-fault condition which
15797	* would have been handled while checking exits due to event delivery.
15798	*/
15799	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15800	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15801	AssertRCReturn(rc, rc);
15802	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15803
15804	#ifdef DEBUG_ramshankar
15805	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15806	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15807	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15808	#endif
15809
15810	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15811	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15812	return VINF_SUCCESS;
15813	}
15814
15815
15816	/**
15817	* VM-exit exception handler for \#PF (Page-fault exception).
15818	*/
15819	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15820	{
15821	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15822	PVM pVM = pVCpu->CTX_SUFF(pVM);
15823	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15824	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15825	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15826	AssertRCReturn(rc, rc);
15827
15828	if (!pVM->hm.s.fNestedPaging)
15829	{ /* likely */ }
15830	else
15831	{
15832	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15833	Assert(pVCpu->hm.s.fUsingDebugLoop);
15834	#endif
15835	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15836	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15837	{
15838	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15839	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15840	}
15841	else
15842	{
15843	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15844	hmR0VmxSetPendingXcptDF(pVCpu);
15845	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15846	}
15847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15848	return rc;
15849	}
15850
15851	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15852	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15853	if (pVmxTransient->fVectoringPF)
15854	{
15855	Assert(pVCpu->hm.s.Event.fPending);
15856	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15857	}
15858
15859	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15860	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15861	AssertRCReturn(rc, rc);
15862
15863	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15864	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15865
15866	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15867	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15868
15869	Log4Func(("#PF: rc=%Rrc\n", rc));
15870	if (rc == VINF_SUCCESS)
15871	{
15872	/*
15873	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15874	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15875	*/
15876	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15877	TRPMResetTrap(pVCpu);
15878	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15879	return rc;
15880	}
15881
15882	if (rc == VINF_EM_RAW_GUEST_TRAP)
15883	{
15884	if (!pVmxTransient->fVectoringDoublePF)
15885	{
15886	/* It's a guest page fault and needs to be reflected to the guest. */
15887	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15888	TRPMResetTrap(pVCpu);
15889	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15890	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15891	uGstErrorCode, pVmxTransient->uExitQual);
15892	}
15893	else
15894	{
15895	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15896	TRPMResetTrap(pVCpu);
15897	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15898	hmR0VmxSetPendingXcptDF(pVCpu);
15899	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15900	}
15901
15902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15903	return VINF_SUCCESS;
15904	}
15905
15906	TRPMResetTrap(pVCpu);
15907	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15908	return rc;
15909	}
15910
15911
15912	/**
15913	* VM-exit helper for LMSW.
15914	*/
15915	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw,
15916	RTGCPTR GCPtrEffDst)
15917	{
15918	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15919	AssertRCReturn(rc, rc);
15920
15921	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
15922	AssertMsg( rcStrict == VINF_SUCCESS
15923	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15924
15925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15926	if (rcStrict == VINF_IEM_RAISED_XCPT)
15927	{
15928	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15929	rcStrict = VINF_SUCCESS;
15930	}
15931
15932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
15933	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15934	return rcStrict;
15935	}
15936
15937
15938	/**
15939	* VM-exit helper for CLTS.
15940	*/
15941	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
15942	{
15943	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15944	AssertRCReturn(rc, rc);
15945
15946	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
15947	AssertMsg( rcStrict == VINF_SUCCESS
15948	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15949
15950	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15951	if (rcStrict == VINF_IEM_RAISED_XCPT)
15952	{
15953	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15954	rcStrict = VINF_SUCCESS;
15955	}
15956
15957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
15958	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15959	return rcStrict;
15960	}
15961
15962
15963	/**
15964	* VM-exit helper for MOV from CRx (CRx read).
15965	*/
15966	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15967	{
15968	Assert(iCrReg < 16);
15969	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15970
15971	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15972	AssertRCReturn(rc, rc);
15973
15974	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
15975	AssertMsg( rcStrict == VINF_SUCCESS
15976	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15977
15978	if (iGReg == X86_GREG_xSP)
15979	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
15980	else
15981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15982	#ifdef VBOX_WITH_STATISTICS
15983	switch (iCrReg)
15984	{
15985	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
15986	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
15987	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
15988	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
15989	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
15990	}
15991	#endif
15992	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
15993	return rcStrict;
15994	}
15995
15996
15997	/**
15998	* VM-exit helper for MOV to CRx (CRx write).
15999	*/
16000	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
16001	{
16002	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16003	AssertRCReturn(rc, rc);
16004
16005	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
16006	AssertMsg( rcStrict == VINF_SUCCESS
16007	\|\| rcStrict == VINF_IEM_RAISED_XCPT
16008	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16009
16010	switch (iCrReg)
16011	{
16012	case 0:
16013	{
16014	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
16015	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
16016	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
16017	break;
16018	}
16019
16020	case 2:
16021	{
16022	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
16023	/* Nothing to do here, CR2 it's not part of the VMCS. */
16024	break;
16025	}
16026
16027	case 3:
16028	{
16029	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
16030	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
16031	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
16032	break;
16033	}
16034
16035	case 4:
16036	{
16037	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
16038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
16039	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
16040	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
16041	break;
16042	}
16043
16044	case 8:
16045	{
16046	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
16047	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
16048	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
16049	break;
16050	}
16051
16052	default:
16053	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
16054	break;
16055	}
16056
16057	if (rcStrict == VINF_IEM_RAISED_XCPT)
16058	{
16059	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16060	rcStrict = VINF_SUCCESS;
16061	}
16062	return rcStrict;
16063	}
16064
16065
16066	/**
16067	* VM-exit helper for handling host NMIs.
16068	*/
16069	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu)
16070	{
16071	VMXDispatchHostNmi();
16072
16073	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
16074	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
16075	return VINF_SUCCESS;
16076	}
16077
16078
16079	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16080	/** @name VMX instruction handlers.
16081	* @{
16082	*/
16083	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16084	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16085	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16086
16087	/**
16088	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16089	*/
16090	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16091	{
16092	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16093
16094	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16095	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16096	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16097	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16098	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16099	AssertRCReturn(rc, rc);
16100
16101	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16102
16103	VMXVEXITINFO ExitInfo;
16104	RT_ZERO(ExitInfo);
16105	ExitInfo.uReason = pVmxTransient->uExitReason;
16106	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16107	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16108	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16109	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16110
16111	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16112	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16113	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16114	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16115	{
16116	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16117	rcStrict = VINF_SUCCESS;
16118	}
16119	return rcStrict;
16120	}
16121
16122
16123	/**
16124	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16125	*/
16126	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16127	{
16128	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16129
16130	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16131	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16132	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16133	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16134	AssertRCReturn(rc, rc);
16135
16136	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16137
16138	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16139	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16140	{
16141	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16142	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16143	}
16144	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16145	return rcStrict;
16146	}
16147
16148
16149	/**
16150	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16151	*/
16152	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16153	{
16154	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16155
16156	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16157	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16158	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16159	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16160	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16161	AssertRCReturn(rc, rc);
16162
16163	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16164
16165	VMXVEXITINFO ExitInfo;
16166	RT_ZERO(ExitInfo);
16167	ExitInfo.uReason = pVmxTransient->uExitReason;
16168	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16169	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16170	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16171	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16172
16173	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16174	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16175	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16176	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16177	{
16178	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16179	rcStrict = VINF_SUCCESS;
16180	}
16181	return rcStrict;
16182	}
16183
16184
16185	/**
16186	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16187	*/
16188	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16189	{
16190	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16191
16192	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16193	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16194	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16195	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16196	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16197	AssertRCReturn(rc, rc);
16198
16199	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16200
16201	VMXVEXITINFO ExitInfo;
16202	RT_ZERO(ExitInfo);
16203	ExitInfo.uReason = pVmxTransient->uExitReason;
16204	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16205	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16206	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16207	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16208
16209	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16210	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16211	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16212	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16213	{
16214	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16215	rcStrict = VINF_SUCCESS;
16216	}
16217	return rcStrict;
16218	}
16219
16220
16221	/**
16222	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16223	*/
16224	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16225	{
16226	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16227
16228	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16229	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16230	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16231	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16232	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16233	AssertRCReturn(rc, rc);
16234
16235	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16236
16237	VMXVEXITINFO ExitInfo;
16238	RT_ZERO(ExitInfo);
16239	ExitInfo.uReason = pVmxTransient->uExitReason;
16240	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16241	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16242	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16243	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16244	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16245
16246	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16247	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16248	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16249	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16250	{
16251	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16252	rcStrict = VINF_SUCCESS;
16253	}
16254	return rcStrict;
16255	}
16256
16257
16258	/**
16259	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16260	*/
16261	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16262	{
16263	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16264
16265	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16266	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16267	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16268	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16269	AssertRCReturn(rc, rc);
16270
16271	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16272
16273	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16274	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16275	{
16276	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16278	}
16279	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16280	return rcStrict;
16281	}
16282
16283
16284	/**
16285	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16286	*/
16287	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16288	{
16289	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16290
16291	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16292	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16293	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16294	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16295	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16296	AssertRCReturn(rc, rc);
16297
16298	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16299
16300	VMXVEXITINFO ExitInfo;
16301	RT_ZERO(ExitInfo);
16302	ExitInfo.uReason = pVmxTransient->uExitReason;
16303	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16304	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16305	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16306	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16307	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16308
16309	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16310	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16311	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16312	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16313	{
16314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16315	rcStrict = VINF_SUCCESS;
16316	}
16317	return rcStrict;
16318	}
16319
16320
16321	/**
16322	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16323	*/
16324	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16325	{
16326	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16327
16328	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16329	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16330	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16331	AssertRCReturn(rc, rc);
16332
16333	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16334
16335	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16336	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16337	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16338	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16339	{
16340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16341	rcStrict = VINF_SUCCESS;
16342	}
16343	return rcStrict;
16344	}
16345
16346
16347	/**
16348	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16349	*/
16350	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16351	{
16352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16353
16354	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16355	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16356	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16357	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16358	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16359	AssertRCReturn(rc, rc);
16360
16361	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16362
16363	VMXVEXITINFO ExitInfo;
16364	RT_ZERO(ExitInfo);
16365	ExitInfo.uReason = pVmxTransient->uExitReason;
16366	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16367	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16368	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16369	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16370
16371	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16372	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16373	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16374	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16375	{
16376	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16377	rcStrict = VINF_SUCCESS;
16378	}
16379	return rcStrict;
16380	}
16381
16382
16383	/**
16384	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16385	*/
16386	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16387	{
16388	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16389
16390	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16391	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16392	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16393	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16394	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16395	AssertRCReturn(rc, rc);
16396
16397	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16398
16399	VMXVEXITINFO ExitInfo;
16400	RT_ZERO(ExitInfo);
16401	ExitInfo.uReason = pVmxTransient->uExitReason;
16402	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16403	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16404	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16405	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16406
16407	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16408	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16409	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16410	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16411	{
16412	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16413	rcStrict = VINF_SUCCESS;
16414	}
16415	return rcStrict;
16416	}
16417
16418	/** @} */
16419
16420	/** @name Nested-guest VM-exit handlers.
16421	* @{
16422	*/
16423	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16424	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16425	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16426
16427	/**
16428	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16429	* Conditional VM-exit.
16430	*/
16431	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16432	{
16433	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16434
16435	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16436	AssertRCReturn(rc, rc);
16437
16438	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16439	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16440	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16441
16442	/*
16443	* Make sure not to use stale/previous VM-exit instruction length since we read the
16444	* instruction length from the VMCS below only for software exceptions and privileged
16445	* software exceptions but we pass it for all exception VM-exits below.
16446	*/
16447	pVmxTransient->cbInstr = 0;
16448
16449	switch (uExtIntType)
16450	{
16451	/*
16452	* Physical NMIs:
16453	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the
16454	* host.
16455	*/
16456	case VMX_EXIT_INT_INFO_TYPE_NMI:
16457	return hmR0VmxExitHostNmi(pVCpu);
16458
16459	/*
16460	* Hardware exceptions,
16461	* Software exceptions,
16462	* Privileged software exceptions:
16463	* Figure out if the exception must be delivered to the guest or the nested-guest.
16464	*
16465	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16466	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16467	* length.
16468	*/
16469	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16470	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16471	{
16472	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16473	RT_FALL_THRU();
16474	}
16475	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16476	{
16477	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16478	AssertRCReturn(rc, rc);
16479
16480	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
16481	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
16482	pVmxTransient->uExitIntErrorCode);
16483	if (fIntercept)
16484	{
16485	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16486	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16487	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16488	AssertRCReturn(rc, rc);
16489
16490	VMXVEXITINFO ExitInfo;
16491	RT_ZERO(ExitInfo);
16492	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16493	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16494
16495	VMXVEXITEVENTINFO ExitEventInfo;
16496	RT_ZERO(ExitEventInfo);
16497	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16498	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16499	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16500	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16501
16502	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16503	}
16504
16505	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
16506	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16507
16508	/* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
16509	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
16510	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16511	return VINF_SUCCESS;
16512	}
16513
16514	/*
16515	* Software interrupts:
16516	* VM-exits cannot be caused by software interrupts.
16517	*
16518	* External interrupts:
16519	* This should only happen when "acknowledge external interrupts on VM-exit"
16520	* control is set. However, we don't set it when executing guests or
16521	* nested-guests. For nested-guests it is emulated while injecting interrupts into
16522	* the guest.
16523	*/
16524	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16525	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16526	default:
16527	{
16528	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16529	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16530	}
16531	}
16532	}
16533
16534
16535	/**
16536	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16537	* Unconditional VM-exit.
16538	*/
16539	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16540	{
16541	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16542	return IEMExecVmxVmexitTripleFault(pVCpu);
16543	}
16544
16545
16546	/**
16547	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16548	*/
16549	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16550	{
16551	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16552
16553	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16554	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16555	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16556	}
16557
16558
16559	/**
16560	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16561	*/
16562	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16563	{
16564	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16565
16566	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16567	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16568	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16569	}
16570
16571
16572	/**
16573	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16574	* Unconditional VM-exit.
16575	*/
16576	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16577	{
16578	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16579
16580	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16581	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16582	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16583	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16584	AssertRCReturn(rc, rc);
16585
16586	VMXVEXITINFO ExitInfo;
16587	RT_ZERO(ExitInfo);
16588	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16589	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16590
16591	VMXVEXITEVENTINFO ExitEventInfo;
16592	RT_ZERO(ExitInfo);
16593	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16594	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16595	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16596	}
16597
16598
16599	/**
16600	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16601	*/
16602	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16603	{
16604	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16605
16606	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16607	{
16608	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16609	AssertRCReturn(rc, rc);
16610	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16611	}
16612	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16613	}
16614
16615
16616	/**
16617	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16618	*/
16619	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16620	{
16621	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16622
16623	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16624	{
16625	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16626	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16627	AssertRCReturn(rc, rc);
16628
16629	VMXVEXITINFO ExitInfo;
16630	RT_ZERO(ExitInfo);
16631	ExitInfo.uReason = pVmxTransient->uExitReason;
16632	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16633	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16634	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16635	}
16636	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16637	}
16638
16639
16640	/**
16641	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16642	*/
16643	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16644	{
16645	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16646
16647	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16648	{
16649	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16650	AssertRCReturn(rc, rc);
16651	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16652	}
16653	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16654	}
16655
16656
16657	/**
16658	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16659	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16660	*/
16661	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16662	{
16663	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16664
16665	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16666	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16667
16668	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16669	AssertRCReturn(rc, rc);
16670
16671	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16672	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16673	uint64_t u64FieldEnc = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16674
16675	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16676	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16677	u64FieldEnc &= UINT64_C(0xffffffff);
16678
16679	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64FieldEnc))
16680	{
16681	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16682	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16683	AssertRCReturn(rc, rc);
16684
16685	VMXVEXITINFO ExitInfo;
16686	RT_ZERO(ExitInfo);
16687	ExitInfo.uReason = pVmxTransient->uExitReason;
16688	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16689	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16690	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16691	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16692	}
16693
16694	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16695	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16696	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16697	}
16698
16699
16700	/**
16701	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16702	*/
16703	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16704	{
16705	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16706
16707	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16708	{
16709	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16710	AssertRCReturn(rc, rc);
16711	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16712	}
16713
16714	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16715	}
16716
16717
16718	/**
16719	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16720	* Conditional VM-exit.
16721	*/
16722	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16723	{
16724	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16725
16726	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16727	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16728	AssertRCReturn(rc, rc);
16729
16730	VBOXSTRICTRC rcStrict;
16731	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16732	switch (uAccessType)
16733	{
16734	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16735	{
16736	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16737	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16738	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16739	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16740	if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
16741	{
16742	VMXVEXITINFO ExitInfo;
16743	RT_ZERO(ExitInfo);
16744	ExitInfo.uReason = pVmxTransient->uExitReason;
16745	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16746	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16747	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16748	}
16749	else
16750	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16751	break;
16752	}
16753
16754	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16755	{
16756	/*
16757	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16758	* CR2 reads do not cause a VM-exit.
16759	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16760	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16761	*/
16762	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16763	if ( iCrReg == 3
16764	\|\| iCrReg == 8)
16765	{
16766	static const uint32_t s_aCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16767	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16768	uint32_t const uIntercept = s_aCrXReadIntercepts[iCrReg];
16769	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16770	{
16771	VMXVEXITINFO ExitInfo;
16772	RT_ZERO(ExitInfo);
16773	ExitInfo.uReason = pVmxTransient->uExitReason;
16774	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16775	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16776	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16777	}
16778	else
16779	{
16780	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16781	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16782	}
16783	}
16784	else
16785	{
16786	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16787	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16788	}
16789	break;
16790	}
16791
16792	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16793	{
16794	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16795	Assert(pVmcsNstGst);
16796	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16797	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16798	if ( (uGstHostMask & X86_CR0_TS)
16799	&& (uReadShadow & X86_CR0_TS))
16800	{
16801	VMXVEXITINFO ExitInfo;
16802	RT_ZERO(ExitInfo);
16803	ExitInfo.uReason = pVmxTransient->uExitReason;
16804	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16805	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16806	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16807	}
16808	else
16809	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16810	break;
16811	}
16812
16813	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16814	{
16815	RTGCPTR GCPtrEffDst;
16816	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16817	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16818	if (fMemOperand)
16819	{
16820	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16821	AssertRCReturn(rc, rc);
16822	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16823	}
16824	else
16825	GCPtrEffDst = NIL_RTGCPTR;
16826
16827	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16828	{
16829	VMXVEXITINFO ExitInfo;
16830	RT_ZERO(ExitInfo);
16831	ExitInfo.uReason = pVmxTransient->uExitReason;
16832	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16833	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16834	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16835	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16836	}
16837	else
16838	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16839	break;
16840	}
16841
16842	default:
16843	{
16844	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16845	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16846	}
16847	}
16848
16849	if (rcStrict == VINF_IEM_RAISED_XCPT)
16850	{
16851	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16852	rcStrict = VINF_SUCCESS;
16853	}
16854	return rcStrict;
16855	}
16856
16857
16858	/**
16859	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16860	* Conditional VM-exit.
16861	*/
16862	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16863	{
16864	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16865
16866	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16867	{
16868	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16869	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16870	AssertRCReturn(rc, rc);
16871
16872	VMXVEXITINFO ExitInfo;
16873	RT_ZERO(ExitInfo);
16874	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16875	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16876	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16877	}
16878	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16879	}
16880
16881
16882	/**
16883	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16884	* Conditional VM-exit.
16885	*/
16886	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16887	{
16888	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16889
16890	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16891	AssertRCReturn(rc, rc);
16892
16893	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16894	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16895	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16896
16897	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16898	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16899	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16900	{
16901	/*
16902	* IN/OUT instruction:
16903	* - Provides VM-exit instruction length.
16904	*
16905	* INS/OUTS instruction:
16906	* - Provides VM-exit instruction length.
16907	* - Provides Guest-linear address.
16908	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16909	*/
16910	PVM pVM = pVCpu->CTX_SUFF(pVM);
16911	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16912	AssertRCReturn(rc, rc);
16913
16914	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16915	pVmxTransient->ExitInstrInfo.u = 0;
16916	pVmxTransient->uGuestLinearAddr = 0;
16917
16918	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16919	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16920	if (fIOString)
16921	{
16922	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16923	if (fVmxInsOutsInfo)
16924	{
16925	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16926	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16927	}
16928	}
16929	AssertRCReturn(rc, rc);
16930
16931	VMXVEXITINFO ExitInfo;
16932	RT_ZERO(ExitInfo);
16933	ExitInfo.uReason = pVmxTransient->uExitReason;
16934	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16935	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16936	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16937	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16938	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16939	}
16940	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16941	}
16942
16943
16944	/**
16945	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16946	*/
16947	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16948	{
16949	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16950
16951	uint32_t fMsrpm;
16952	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16953	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16954	else
16955	fMsrpm = VMXMSRPM_EXIT_RD;
16956
16957	if (fMsrpm & VMXMSRPM_EXIT_RD)
16958	{
16959	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16960	AssertRCReturn(rc, rc);
16961	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16962	}
16963	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16964	}
16965
16966
16967	/**
16968	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16969	*/
16970	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16971	{
16972	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16973
16974	uint32_t fMsrpm;
16975	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16976	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16977	else
16978	fMsrpm = VMXMSRPM_EXIT_WR;
16979
16980	if (fMsrpm & VMXMSRPM_EXIT_WR)
16981	{
16982	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16983	AssertRCReturn(rc, rc);
16984	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16985	}
16986	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16987	}
16988
16989
16990	/**
16991	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16992	*/
16993	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16994	{
16995	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16996
16997	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16998	{
16999	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17000	AssertRCReturn(rc, rc);
17001	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17002	}
17003	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17004	}
17005
17006
17007	/**
17008	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17009	* VM-exit.
17010	*/
17011	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17012	{
17013	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17014
17015	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17016	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17017	}
17018
17019
17020	/**
17021	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17022	*/
17023	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17024	{
17025	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17026
17027	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17028	{
17029	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17030	AssertRCReturn(rc, rc);
17031	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17032	}
17033	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17034	}
17035
17036
17037	/**
17038	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17039	*/
17040	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17041	{
17042	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17043
17044	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17045	* PAUSE when executing a nested-guest? If it does not, we would not need
17046	* to check for the intercepts here. Just call VM-exit... */
17047
17048	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17049	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17050	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17051	{
17052	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17053	AssertRCReturn(rc, rc);
17054	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17055	}
17056	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17057	}
17058
17059
17060	/**
17061	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17062	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17063	*/
17064	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17065	{
17066	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17067
17068	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17069	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17070	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17071	}
17072
17073
17074	/**
17075	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17076	* VM-exit.
17077	*/
17078	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17079	{
17080	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17081
17082	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17083	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17084	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17085	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17086	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17087	AssertRCReturn(rc, rc);
17088
17089	VMXVEXITINFO ExitInfo;
17090	RT_ZERO(ExitInfo);
17091	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17092	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17093
17094	VMXVEXITEVENTINFO ExitEventInfo;
17095	RT_ZERO(ExitInfo);
17096	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17097	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17098	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17099	}
17100
17101
17102	/**
17103	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17104	* Conditional VM-exit.
17105	*/
17106	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17107	{
17108	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17109
17110	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17111	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17112	AssertRCReturn(rc, rc);
17113
17114	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17115	}
17116
17117
17118	/**
17119	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17120	* Conditional VM-exit.
17121	*/
17122	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17123	{
17124	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17125
17126	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17127	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17128	AssertRCReturn(rc, rc);
17129
17130	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17131	}
17132
17133
17134	/**
17135	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17136	*/
17137	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17138	{
17139	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17140
17141	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17142	{
17143	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17144	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17145	AssertRCReturn(rc, rc);
17146	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17147	}
17148	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17149	}
17150
17151
17152	/**
17153	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17154	*/
17155	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17156	{
17157	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17158
17159	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17160	{
17161	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17162	AssertRCReturn(rc, rc);
17163	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17164	}
17165	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17166	}
17167
17168
17169	/**
17170	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17171	*/
17172	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17173	{
17174	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17175
17176	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17177	{
17178	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17179	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17180	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17181	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17182	AssertRCReturn(rc, rc);
17183
17184	VMXVEXITINFO ExitInfo;
17185	RT_ZERO(ExitInfo);
17186	ExitInfo.uReason = pVmxTransient->uExitReason;
17187	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17188	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17189	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17190	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17191	}
17192	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17193	}
17194
17195
17196	/**
17197	* Nested-guest VM-exit handler for invalid-guest state
17198	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17199	*/
17200	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17201	{
17202	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17203
17204	/*
17205	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17206	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17207	* Handle it like it's in an invalid guest state of the outer guest.
17208	*
17209	* When the fast path is implemented, this should be changed to cause the corresponding
17210	* nested-guest VM-exit.
17211	*/
17212	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17213	}
17214
17215
17216	/**
17217	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17218	* and only provide the instruction length.
17219	*
17220	* Unconditional VM-exit.
17221	*/
17222	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17223	{
17224	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17225
17226	#ifdef VBOX_STRICT
17227	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17228	switch (pVmxTransient->uExitReason)
17229	{
17230	case VMX_EXIT_ENCLS:
17231	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17232	break;
17233
17234	case VMX_EXIT_VMFUNC:
17235	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17236	break;
17237	}
17238	#endif
17239
17240	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17241	AssertRCReturn(rc, rc);
17242	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17243	}
17244
17245
17246	/**
17247	* Nested-guest VM-exit handler for instructions that provide instruction length as
17248	* well as more information.
17249	*
17250	* Unconditional VM-exit.
17251	*/
17252	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17253	{
17254	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17255
17256	#ifdef VBOX_STRICT
17257	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17258	switch (pVmxTransient->uExitReason)
17259	{
17260	case VMX_EXIT_GDTR_IDTR_ACCESS:
17261	case VMX_EXIT_LDTR_TR_ACCESS:
17262	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17263	break;
17264
17265	case VMX_EXIT_RDRAND:
17266	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17267	break;
17268
17269	case VMX_EXIT_RDSEED:
17270	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17271	break;
17272
17273	case VMX_EXIT_XSAVES:
17274	case VMX_EXIT_XRSTORS:
17275	/** @todo NSTVMX: Verify XSS-bitmap. */
17276	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17277	break;
17278
17279	case VMX_EXIT_UMWAIT:
17280	case VMX_EXIT_TPAUSE:
17281	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17282	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17283	break;
17284	}
17285	#endif
17286
17287	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17288	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17289	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17290	AssertRCReturn(rc, rc);
17291
17292	VMXVEXITINFO ExitInfo;
17293	RT_ZERO(ExitInfo);
17294	ExitInfo.uReason = pVmxTransient->uExitReason;
17295	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17296	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17297	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17298	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17299	}
17300
17301	/** @} */
17302
17303	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17304

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79232

Download in other formats: