HMVMXR0.cpp@ 78772

Last change on this file since 78772 was 78772, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Added a separate LMSW VM-exit handler after decoding phase is done. VM-exit exception handler helpers should return rcStrict rather than int. Renamed VMX_EXIT_QUAL_CRX_LMSW_OP to VMX_EXIT_QUAL_CRX_LMSW_OP_MEM to reflect 1 is memory operand.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 668.0 KB

Line
1	/* $Id: HMVMXR0.cpp 78772 2019-05-27 05:22:48Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
366	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369	static FNVMXEXITHANDLER hmR0VmxExitHlt;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
371	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
376	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
377	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
378	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
379	static FNVMXEXITHANDLER hmR0VmxExitVmread;
380	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
381	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
382	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
383	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
384	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
385	#endif
386	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
387	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
388	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
389	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
390	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
391	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
392	static FNVMXEXITHANDLER hmR0VmxExitMwait;
393	static FNVMXEXITHANDLER hmR0VmxExitMtf;
394	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395	static FNVMXEXITHANDLER hmR0VmxExitPause;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
404	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
410	/** @} */
411
412	/** @name Helpers for hardware exceptions VM-exit handlers.
413	* @{
414	*/
415	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
416	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
417	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
418	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
419	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
420	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
421	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
423	/** @} */
424
425
426	/*********************************************************************************************************************************
427	* Global Variables *
428	*********************************************************************************************************************************/
429	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
430	static const uint32_t g_aVmcsCacheSegBase[] =
431	{
432	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
433	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
434	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
435	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
436	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
437	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
438	};
439	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
440	#endif
441	static const uint32_t g_aVmcsSegBase[] =
442	{
443	VMX_VMCS_GUEST_ES_BASE,
444	VMX_VMCS_GUEST_CS_BASE,
445	VMX_VMCS_GUEST_SS_BASE,
446	VMX_VMCS_GUEST_DS_BASE,
447	VMX_VMCS_GUEST_FS_BASE,
448	VMX_VMCS_GUEST_GS_BASE
449	};
450	static const uint32_t g_aVmcsSegSel[] =
451	{
452	VMX_VMCS16_GUEST_ES_SEL,
453	VMX_VMCS16_GUEST_CS_SEL,
454	VMX_VMCS16_GUEST_SS_SEL,
455	VMX_VMCS16_GUEST_DS_SEL,
456	VMX_VMCS16_GUEST_FS_SEL,
457	VMX_VMCS16_GUEST_GS_SEL
458	};
459	static const uint32_t g_aVmcsSegLimit[] =
460	{
461	VMX_VMCS32_GUEST_ES_LIMIT,
462	VMX_VMCS32_GUEST_CS_LIMIT,
463	VMX_VMCS32_GUEST_SS_LIMIT,
464	VMX_VMCS32_GUEST_DS_LIMIT,
465	VMX_VMCS32_GUEST_FS_LIMIT,
466	VMX_VMCS32_GUEST_GS_LIMIT
467	};
468	static const uint32_t g_aVmcsSegAttr[] =
469	{
470	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
471	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
472	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
473	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
474	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
475	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
476	};
477	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
478	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
479	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
480	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
481
482	#ifdef HMVMX_USE_FUNCTION_TABLE
483	/**
484	* VMX_EXIT dispatch table.
485	*/
486	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
487	{
488	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
489	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
490	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
491	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
492	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
493	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
494	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
495	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
496	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
497	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
498	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
499	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
500	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
501	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
502	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
503	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
504	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
505	/* 17 VMX_EXIT_RSM */ hmR0VmxExitSetPendingXcptUD,
506	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
507	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
508	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
509	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
510	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
511	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
512	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
513	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
514	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
515	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
516	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
517	#else
518	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
519	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
520	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
521	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
522	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
523	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
524	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
525	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
526	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
527	#endif
528	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
529	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
530	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
531	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
532	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
533	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
534	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
535	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
536	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
537	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
538	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
539	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
540	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
541	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
542	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
543	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
544	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
545	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
546	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
547	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
548	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
549	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
550	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
551	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
552	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
554	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
555	#else
556	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
557	#endif
558	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
559	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
560	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
561	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
562	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
563	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
564	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitSetPendingXcptUD,
565	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitSetPendingXcptUD,
566	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
567	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
568	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
569	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
570	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
571	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitSetPendingXcptUD,
572	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitSetPendingXcptUD,
573	};
574	#endif /* HMVMX_USE_FUNCTION_TABLE */
575
576	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
577	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
578	{
579	/* 0 */ "(Not Used)",
580	/* 1 */ "VMCALL executed in VMX root operation.",
581	/* 2 */ "VMCLEAR with invalid physical address.",
582	/* 3 */ "VMCLEAR with VMXON pointer.",
583	/* 4 */ "VMLAUNCH with non-clear VMCS.",
584	/* 5 */ "VMRESUME with non-launched VMCS.",
585	/* 6 */ "VMRESUME after VMXOFF",
586	/* 7 */ "VM-entry with invalid control fields.",
587	/* 8 */ "VM-entry with invalid host state fields.",
588	/* 9 */ "VMPTRLD with invalid physical address.",
589	/* 10 */ "VMPTRLD with VMXON pointer.",
590	/* 11 */ "VMPTRLD with incorrect revision identifier.",
591	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
592	/* 13 */ "VMWRITE to read-only VMCS component.",
593	/* 14 */ "(Not Used)",
594	/* 15 */ "VMXON executed in VMX root operation.",
595	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
596	/* 17 */ "VM-entry with non-launched executing VMCS.",
597	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
598	/* 19 */ "VMCALL with non-clear VMCS.",
599	/* 20 */ "VMCALL with invalid VM-exit control fields.",
600	/* 21 */ "(Not Used)",
601	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
602	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
603	/* 24 */ "VMCALL with invalid SMM-monitor features.",
604	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
605	/* 26 */ "VM-entry with events blocked by MOV SS.",
606	/* 27 */ "(Not Used)",
607	/* 28 */ "Invalid operand to INVEPT/INVVPID."
608	};
609	#endif /* VBOX_STRICT */
610
611
612	/**
613	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
614	*
615	* Any bit set in this mask is owned by the host/hypervisor and would cause a
616	* VM-exit when modified by the guest.
617	*
618	* @returns The static CR0 guest/host mask.
619	* @param pVCpu The cross context virtual CPU structure.
620	*/
621	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
622	{
623	/*
624	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
625	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
626	*/
627	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
628	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
629	* and @bugref{6944}. */
630	PVM pVM = pVCpu->CTX_SUFF(pVM);
631	return ( X86_CR0_PE
632	\| X86_CR0_NE
633	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
634	\| X86_CR0_PG
635	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
636	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
637	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
638	}
639
640
641	/**
642	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
643	*
644	* Any bit set in this mask is owned by the host/hypervisor and would cause a
645	* VM-exit when modified by the guest.
646	*
647	* @returns The static CR4 guest/host mask.
648	* @param pVCpu The cross context virtual CPU structure.
649	*/
650	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
651	{
652	/*
653	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
654	* these bits are reserved on hardware that does not support them. Since the
655	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
656	* these bits and handle it depending on whether we expose them to the guest.
657	*/
658	PVM pVM = pVCpu->CTX_SUFF(pVM);
659	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
660	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
661	return ( X86_CR4_VMXE
662	\| X86_CR4_VME
663	\| X86_CR4_PAE
664	\| X86_CR4_PGE
665	\| X86_CR4_PSE
666	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
667	\| (fPcid ? X86_CR4_PCIDE : 0));
668	}
669
670
671	/**
672	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
673	* area.
674	*
675	* @returns @c true if it's different, @c false otherwise.
676	* @param pVmcsInfo The VMCS info. object.
677	*/
678	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
679	{
680	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
681	&& pVmcsInfo->pvGuestMsrStore);
682	}
683
684
685	/**
686	* Adds one or more exceptions to the exception bitmap and commits it to the current
687	* VMCS.
688	*
689	* @returns VBox status code.
690	* @param pVmxTransient The VMX-transient structure.
691	* @param uXcptMask The exception(s) to add.
692	*/
693	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
694	{
695	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
696	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
697	if ((uXcptBitmap & uXcptMask) != uXcptMask)
698	{
699	uXcptBitmap \|= uXcptMask;
700	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
701	AssertRCReturn(rc, rc);
702	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
703	}
704	return VINF_SUCCESS;
705	}
706
707
708	/**
709	* Adds an exception to the exception bitmap and commits it to the current VMCS.
710	*
711	* @returns VBox status code.
712	* @param pVmxTransient The VMX-transient structure.
713	* @param uXcpt The exception to add.
714	*/
715	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
716	{
717	Assert(uXcpt <= X86_XCPT_LAST);
718	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
719	}
720
721
722	/**
723	* Remove one or more exceptions from the exception bitmap and commits it to the
724	* current VMCS.
725	*
726	* This takes care of not removing the exception intercept if a nested-guest
727	* requires the exception to be intercepted.
728	*
729	* @returns VBox status code.
730	* @param pVCpu The cross context virtual CPU structure.
731	* @param pVmxTransient The VMX-transient structure.
732	* @param uXcptMask The exception(s) to remove.
733	*/
734	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
735	{
736	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
737	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
738	if (u32XcptBitmap & uXcptMask)
739	{
740	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
741	if (!pVmxTransient->fIsNestedGuest)
742	{ /* likely */ }
743	else
744	{
745	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
746	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
747	}
748	#endif
749	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
750	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
751	\| RT_BIT(X86_XCPT_DE)
752	\| RT_BIT(X86_XCPT_NM)
753	\| RT_BIT(X86_XCPT_TS)
754	\| RT_BIT(X86_XCPT_UD)
755	\| RT_BIT(X86_XCPT_NP)
756	\| RT_BIT(X86_XCPT_SS)
757	\| RT_BIT(X86_XCPT_GP)
758	\| RT_BIT(X86_XCPT_PF)
759	\| RT_BIT(X86_XCPT_MF));
760	#elif defined(HMVMX_ALWAYS_TRAP_PF)
761	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
762	#endif
763	if (uXcptMask)
764	{
765	/* Validate we are not removing any essential exception intercepts. */
766	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
767	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
768	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
769
770	/* Remove it from the exception bitmap. */
771	u32XcptBitmap &= ~uXcptMask;
772
773	/* Commit and update the cache if necessary. */
774	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
775	{
776	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
777	AssertRCReturn(rc, rc);
778	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
779	}
780	}
781	}
782	return VINF_SUCCESS;
783	}
784
785
786	/**
787	* Remove an exceptions from the exception bitmap and commits it to the current
788	* VMCS.
789	*
790	* @returns VBox status code.
791	* @param pVCpu The cross context virtual CPU structure.
792	* @param pVmxTransient The VMX-transient structure.
793	* @param uXcpt The exception to remove.
794	*/
795	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
796	{
797	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
798	}
799
800
801	/**
802	* Loads the VMCS specified by the VMCS info. object.
803	*
804	* @returns VBox status code.
805	* @param pVmcsInfo The VMCS info. object.
806	*/
807	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
808	{
809	Assert(pVmcsInfo);
810	Assert(pVmcsInfo->HCPhysVmcs);
811	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
814	{
815	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
816	if (RT_SUCCESS(rc))
817	{
818	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
819	return VINF_SUCCESS;
820	}
821	return rc;
822	}
823	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
824	}
825
826
827	/**
828	* Clears the VMCS specified by the VMCS info. object.
829	*
830	* @returns VBox status code.
831	* @param pVmcsInfo The VMCS info. object.
832	*/
833	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
834	{
835	Assert(pVmcsInfo);
836	Assert(pVmcsInfo->HCPhysVmcs);
837	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
838
839	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
840	if (RT_SUCCESS(rc))
841	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
842	return rc;
843	}
844
845
846	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
847	/**
848	* Switches the current VMCS to the one specified.
849	*
850	* @returns VBox status code.
851	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
852	* @param pVmcsInfoTo The VMCS info. object we are switching to.
853	*
854	* @remarks Called with interrupts disabled.
855	*/
856	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
857	{
858	Assert(pVmcsInfoFrom);
859	Assert(pVmcsInfoTo);
860
861	/*
862	* Clear the VMCS we are switching out if it has not already been cleared.
863	* This will sync any CPU internal data back to the VMCS.
864	*/
865	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
866	{
867	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
868	if (RT_SUCCESS(rc))
869	{ /* likely */ }
870	else
871	return rc;
872	}
873
874	/*
875	* Clear the VMCS we are switching to if it has not already been cleared.
876	* This will initialize the VMCS launch state to "clear" required for loading it.
877	*
878	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
879	*/
880	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
881	{
882	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
883	if (RT_SUCCESS(rc))
884	{ /* likely */ }
885	else
886	return rc;
887	}
888
889	/*
890	* Finally, load the VMCS we are switching to.
891	*/
892	return hmR0VmxLoadVmcs(pVmcsInfoTo);
893	}
894	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
895
896
897	/**
898	* Updates the VM's last error record.
899	*
900	* If there was a VMX instruction error, reads the error data from the VMCS and
901	* updates VCPU's last error record as well.
902	*
903	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
904	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
905	* VERR_VMX_INVALID_VMCS_FIELD.
906	* @param rc The error code.
907	*/
908	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
909	{
910	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
911	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
912	{
913	AssertPtrReturnVoid(pVCpu);
914	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
915	}
916	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
917	}
918
919
920	#ifdef VBOX_STRICT
921	/**
922	* Reads the VM-entry interruption-information field from the VMCS into the VMX
923	* transient structure.
924	*
925	* @returns VBox status code.
926	* @param pVmxTransient The VMX-transient structure.
927	*
928	* @remarks No-long-jump zone!!!
929	*/
930	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
931	{
932	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
933	AssertRCReturn(rc, rc);
934	return VINF_SUCCESS;
935	}
936
937
938	/**
939	* Reads the VM-entry exception error code field from the VMCS into
940	* the VMX transient structure.
941	*
942	* @returns VBox status code.
943	* @param pVmxTransient The VMX-transient structure.
944	*
945	* @remarks No-long-jump zone!!!
946	*/
947	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
948	{
949	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
950	AssertRCReturn(rc, rc);
951	return VINF_SUCCESS;
952	}
953
954
955	/**
956	* Reads the VM-entry exception error code field from the VMCS into
957	* the VMX transient structure.
958	*
959	* @returns VBox status code.
960	* @param pVmxTransient The VMX-transient structure.
961	*
962	* @remarks No-long-jump zone!!!
963	*/
964	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
965	{
966	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
967	AssertRCReturn(rc, rc);
968	return VINF_SUCCESS;
969	}
970	#endif /* VBOX_STRICT */
971
972
973	/**
974	* Reads the VM-exit interruption-information field from the VMCS into the VMX
975	* transient structure.
976	*
977	* @returns VBox status code.
978	* @param pVmxTransient The VMX-transient structure.
979	*/
980	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
981	{
982	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
983	{
984	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
985	AssertRCReturn(rc,rc);
986	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
987	}
988	return VINF_SUCCESS;
989	}
990
991
992	/**
993	* Reads the VM-exit interruption error code from the VMCS into the VMX
994	* transient structure.
995	*
996	* @returns VBox status code.
997	* @param pVmxTransient The VMX-transient structure.
998	*/
999	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1000	{
1001	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1002	{
1003	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1004	AssertRCReturn(rc, rc);
1005	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1006	}
1007	return VINF_SUCCESS;
1008	}
1009
1010
1011	/**
1012	* Reads the VM-exit instruction length field from the VMCS into the VMX
1013	* transient structure.
1014	*
1015	* @returns VBox status code.
1016	* @param pVmxTransient The VMX-transient structure.
1017	*/
1018	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1019	{
1020	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1021	{
1022	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1023	AssertRCReturn(rc, rc);
1024	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1025	}
1026	return VINF_SUCCESS;
1027	}
1028
1029
1030	/**
1031	* Reads the VM-exit instruction-information field from the VMCS into
1032	* the VMX transient structure.
1033	*
1034	* @returns VBox status code.
1035	* @param pVmxTransient The VMX-transient structure.
1036	*/
1037	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1038	{
1039	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1040	{
1041	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1042	AssertRCReturn(rc, rc);
1043	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1044	}
1045	return VINF_SUCCESS;
1046	}
1047
1048
1049	/**
1050	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1051	*
1052	* @returns VBox status code.
1053	* @param pVCpu The cross context virtual CPU structure of the
1054	* calling EMT. (Required for the VMCS cache case.)
1055	* @param pVmxTransient The VMX-transient structure.
1056	*/
1057	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1058	{
1059	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1060	{
1061	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1062	AssertRCReturn(rc, rc);
1063	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1064	}
1065	return VINF_SUCCESS;
1066	}
1067
1068
1069	/**
1070	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1071	*
1072	* @returns VBox status code.
1073	* @param pVCpu The cross context virtual CPU structure of the
1074	* calling EMT. (Required for the VMCS cache case.)
1075	* @param pVmxTransient The VMX-transient structure.
1076	*/
1077	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1078	{
1079	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1080	{
1081	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1082	AssertRCReturn(rc, rc);
1083	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1084	}
1085	return VINF_SUCCESS;
1086	}
1087
1088
1089	/**
1090	* Reads the IDT-vectoring information field from the VMCS into the VMX
1091	* transient structure.
1092	*
1093	* @returns VBox status code.
1094	* @param pVmxTransient The VMX-transient structure.
1095	*
1096	* @remarks No-long-jump zone!!!
1097	*/
1098	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1099	{
1100	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1101	{
1102	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1103	AssertRCReturn(rc, rc);
1104	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1105	}
1106	return VINF_SUCCESS;
1107	}
1108
1109
1110	/**
1111	* Reads the IDT-vectoring error code from the VMCS into the VMX
1112	* transient structure.
1113	*
1114	* @returns VBox status code.
1115	* @param pVmxTransient The VMX-transient structure.
1116	*/
1117	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1118	{
1119	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1120	{
1121	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1122	AssertRCReturn(rc, rc);
1123	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1124	}
1125	return VINF_SUCCESS;
1126	}
1127
1128
1129	/**
1130	* Enters VMX root mode operation on the current CPU.
1131	*
1132	* @returns VBox status code.
1133	* @param pVM The cross context VM structure. Can be
1134	* NULL, after a resume.
1135	* @param HCPhysCpuPage Physical address of the VMXON region.
1136	* @param pvCpuPage Pointer to the VMXON region.
1137	*/
1138	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1139	{
1140	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1141	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1142	Assert(pvCpuPage);
1143	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1144
1145	if (pVM)
1146	{
1147	/* Write the VMCS revision identifier to the VMXON region. */
1148	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1149	}
1150
1151	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1152	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1153
1154	/* Enable the VMX bit in CR4 if necessary. */
1155	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1156
1157	/* Enter VMX root mode. */
1158	int rc = VMXEnable(HCPhysCpuPage);
1159	if (RT_FAILURE(rc))
1160	{
1161	if (!(uOldCr4 & X86_CR4_VMXE))
1162	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1163
1164	if (pVM)
1165	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1166	}
1167
1168	/* Restore interrupts. */
1169	ASMSetFlags(fEFlags);
1170	return rc;
1171	}
1172
1173
1174	/**
1175	* Exits VMX root mode operation on the current CPU.
1176	*
1177	* @returns VBox status code.
1178	*/
1179	static int hmR0VmxLeaveRootMode(void)
1180	{
1181	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1182
1183	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1184	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1185
1186	/* If we're for some reason not in VMX root mode, then don't leave it. */
1187	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1188
1189	int rc;
1190	if (uHostCR4 & X86_CR4_VMXE)
1191	{
1192	/* Exit VMX root mode and clear the VMX bit in CR4. */
1193	VMXDisable();
1194	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1195	rc = VINF_SUCCESS;
1196	}
1197	else
1198	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1199
1200	/* Restore interrupts. */
1201	ASMSetFlags(fEFlags);
1202	return rc;
1203	}
1204
1205
1206	/**
1207	* Allocates and maps a physically contiguous page. The allocated page is
1208	* zero'd out (used by various VT-x structures).
1209	*
1210	* @returns IPRT status code.
1211	* @param pMemObj Pointer to the ring-0 memory object.
1212	* @param ppVirt Where to store the virtual address of the
1213	* allocation.
1214	* @param pHCPhys Where to store the physical address of the
1215	* allocation.
1216	*/
1217	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1218	{
1219	AssertPtr(pMemObj);
1220	AssertPtr(ppVirt);
1221	AssertPtr(pHCPhys);
1222	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1223	if (RT_FAILURE(rc))
1224	return rc;
1225	ppVirt = RTR0MemObjAddress(pMemObj);
1226	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1227	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1228	return VINF_SUCCESS;
1229	}
1230
1231
1232	/**
1233	* Frees and unmaps an allocated, physical page.
1234	*
1235	* @param pMemObj Pointer to the ring-0 memory object.
1236	* @param ppVirt Where to re-initialize the virtual address of
1237	* allocation as 0.
1238	* @param pHCPhys Where to re-initialize the physical address of the
1239	* allocation as 0.
1240	*/
1241	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1242	{
1243	AssertPtr(pMemObj);
1244	AssertPtr(ppVirt);
1245	AssertPtr(pHCPhys);
1246	/* NULL is valid, accepted and ignored by the free function below. */
1247	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1248	*pMemObj = NIL_RTR0MEMOBJ;
1249	*ppVirt = NULL;
1250	*pHCPhys = NIL_RTHCPHYS;
1251	}
1252
1253
1254	/**
1255	* Initializes a VMCS info. object.
1256	*
1257	* @param pVmcsInfo The VMCS info. object.
1258	*/
1259	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1260	{
1261	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1262
1263	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1265	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1266	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1267	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1268	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1272	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1273	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1274	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1275	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1276	}
1277
1278
1279	/**
1280	* Frees the VT-x structures for a VMCS info. object.
1281	*
1282	* @param pVM The cross context VM structure.
1283	* @param pVmcsInfo The VMCS info. object.
1284	*/
1285	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1286	{
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1288
1289	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1291
1292	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1293	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1294	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1295
1296	hmR0VmxInitVmcsInfo(pVmcsInfo);
1297	}
1298
1299
1300	/**
1301	* Allocates the VT-x structures for a VMCS info. object.
1302	*
1303	* @returns VBox status code.
1304	* @param pVCpu The cross context virtual CPU structure.
1305	* @param pVmcsInfo The VMCS info. object.
1306	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1307	*/
1308	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1309	{
1310	PVM pVM = pVCpu->CTX_SUFF(pVM);
1311
1312	/* Allocate the guest VM control structure (VMCS). */
1313	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1314	if (RT_SUCCESS(rc))
1315	{
1316	if (!fIsNstGstVmcs)
1317	{
1318	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1319	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1320	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1321	{
1322	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1323	NULL /* pR3Ptr /, NULL / pRCPtr */);
1324	}
1325	}
1326	else
1327	{
1328	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1329	Assert(!pVmcsInfo->pbVirtApic);
1330	}
1331
1332	if (RT_SUCCESS(rc))
1333	{
1334	/*
1335	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1336	* transparent accesses of specific MSRs.
1337	*
1338	* If the condition for enabling MSR bitmaps changes here, don't forget to
1339	* update HMIsMsrBitmapActive().
1340	*
1341	* We don't share MSR bitmaps between the guest and nested-guest as we then
1342	* don't need to care about carefully restoring the guest MSR bitmap.
1343	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1344	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1345	*/
1346	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1347	{
1348	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1349	if (RT_SUCCESS(rc))
1350	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1351	}
1352
1353	if (RT_SUCCESS(rc))
1354	{
1355	/*
1356	* Allocate the VM-entry MSR-load area for the guest MSRs.
1357	*
1358	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1359	* the guest and nested-guest.
1360	*/
1361	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1362	&pVmcsInfo->HCPhysGuestMsrLoad);
1363	if (RT_SUCCESS(rc))
1364	{
1365	/*
1366	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1367	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1368	*/
1369	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1370	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1371	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1372
1373	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1374	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1375	&pVmcsInfo->HCPhysHostMsrLoad);
1376	}
1377	}
1378	}
1379	}
1380
1381	return rc;
1382	}
1383
1384
1385	/**
1386	* Free all VT-x structures for the VM.
1387	*
1388	* @returns IPRT status code.
1389	* @param pVM The cross context VM structure.
1390	*/
1391	static void hmR0VmxStructsFree(PVM pVM)
1392	{
1393	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1394	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1395	#endif
1396	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1397
1398	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1399	{
1400	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1401	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1402	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1403	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1404	if (pVM->cpum.ro.GuestFeatures.fVmx)
1405	{
1406	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1407	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1408	}
1409	#endif
1410	}
1411	}
1412
1413
1414	/**
1415	* Allocate all VT-x structures for the VM.
1416	*
1417	* @returns IPRT status code.
1418	* @param pVM The cross context VM structure.
1419	*/
1420	static int hmR0VmxStructsAlloc(PVM pVM)
1421	{
1422	/*
1423	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1424	* The VMCS size cannot be more than 4096 bytes.
1425	*
1426	* See Intel spec. Appendix A.1 "Basic VMX Information".
1427	*/
1428	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1429	if (cbVmcs <= X86_PAGE_4K_SIZE)
1430	{ /* likely */ }
1431	else
1432	{
1433	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1434	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1435	}
1436
1437	/*
1438	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1439	*/
1440	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1441	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1442	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1443	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1444	#endif
1445
1446	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1447	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1448	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1449
1450	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1451	{
1452	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1453	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1454	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1455	}
1456
1457	/*
1458	* Allocate per-VM VT-x structures.
1459	*/
1460	int rc = VINF_SUCCESS;
1461	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1462	/* Allocate crash-dump magic scratch page. */
1463	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1464	if (RT_FAILURE(rc))
1465	{
1466	hmR0VmxStructsFree(pVM);
1467	return rc;
1468	}
1469	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1470	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1471	#endif
1472
1473	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1474	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1475	{
1476	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1477	&pVM->hm.s.vmx.HCPhysApicAccess);
1478	if (RT_FAILURE(rc))
1479	{
1480	hmR0VmxStructsFree(pVM);
1481	return rc;
1482	}
1483	}
1484
1485	/*
1486	* Initialize per-VCPU VT-x structures.
1487	*/
1488	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1489	{
1490	/* Allocate the guest VMCS structures. */
1491	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1492	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1493	if (RT_SUCCESS(rc))
1494	{
1495	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1496	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1497	if (pVM->cpum.ro.GuestFeatures.fVmx)
1498	{
1499	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1500	if (RT_SUCCESS(rc))
1501	{ /* likely */ }
1502	else
1503	break;
1504	}
1505	#endif
1506	}
1507	else
1508	break;
1509	}
1510
1511	if (RT_FAILURE(rc))
1512	{
1513	hmR0VmxStructsFree(pVM);
1514	return rc;
1515	}
1516
1517	return VINF_SUCCESS;
1518	}
1519
1520
1521	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1522	/**
1523	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1524	*
1525	* @returns @c true if the MSR is intercepted, @c false otherwise.
1526	* @param pvMsrBitmap The MSR bitmap.
1527	* @param offMsr The MSR byte offset.
1528	* @param iBit The bit offset from the byte offset.
1529	*/
1530	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1531	{
1532	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1533	Assert(pbMsrBitmap);
1534	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1535	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1536	}
1537	#endif
1538
1539
1540	/**
1541	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1542	*
1543	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1544	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1545	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1546	* the read/write access of this MSR.
1547	*
1548	* @param pVCpu The cross context virtual CPU structure.
1549	* @param pVmcsInfo The VMCS info. object.
1550	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1551	* @param idMsr The MSR value.
1552	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1553	* include both a read -and- a write permission!
1554	*
1555	* @sa CPUMGetVmxMsrPermission.
1556	* @remarks Can be called with interrupts disabled.
1557	*/
1558	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1559	{
1560	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1561	Assert(pbMsrBitmap);
1562	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1563
1564	/*
1565	* MSR-bitmap Layout:
1566	* Byte index MSR range Interpreted as
1567	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1568	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1569	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1570	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1571	*
1572	* A bit corresponding to an MSR within the above range causes a VM-exit
1573	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1574	* the MSR range, it always cause a VM-exit.
1575	*
1576	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1577	*/
1578	uint16_t const offBitmapRead = 0;
1579	uint16_t const offBitmapWrite = 0x800;
1580	uint16_t offMsr;
1581	int32_t iBit;
1582	if (idMsr <= UINT32_C(0x00001fff))
1583	{
1584	offMsr = 0;
1585	iBit = idMsr;
1586	}
1587	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1588	{
1589	offMsr = 0x400;
1590	iBit = idMsr - UINT32_C(0xc0000000);
1591	}
1592	else
1593	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1594
1595	/*
1596	* Set the MSR read permission.
1597	*/
1598	uint16_t const offMsrRead = offBitmapRead + offMsr;
1599	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1600	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1601	{
1602	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1603	bool const fClear = !fIsNstGstVmcs ? true
1604	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1605	#else
1606	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1607	bool const fClear = true;
1608	#endif
1609	if (fClear)
1610	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1611	}
1612	else
1613	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1614
1615	/*
1616	* Set the MSR write permission.
1617	*/
1618	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1619	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1620	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1621	{
1622	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1623	bool const fClear = !fIsNstGstVmcs ? true
1624	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1625	#else
1626	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1627	bool const fClear = true;
1628	#endif
1629	if (fClear)
1630	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1631	}
1632	else
1633	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1634	}
1635
1636
1637	/**
1638	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1639	* area.
1640	*
1641	* @returns VBox status code.
1642	* @param pVCpu The cross context virtual CPU structure.
1643	* @param pVmcsInfo The VMCS info. object.
1644	* @param cMsrs The number of MSRs.
1645	*/
1646	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1647	{
1648	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1649	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1650	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1651	{
1652	/* Commit the MSR counts to the VMCS and update the cache. */
1653	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1654	{
1655	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1657	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1658	AssertRCReturn(rc, rc);
1659
1660	pVmcsInfo->cEntryMsrLoad = cMsrs;
1661	pVmcsInfo->cExitMsrStore = cMsrs;
1662	pVmcsInfo->cExitMsrLoad = cMsrs;
1663	}
1664	return VINF_SUCCESS;
1665	}
1666
1667	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1668	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1669	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1670	}
1671
1672
1673	/**
1674	* Adds a new (or updates the value of an existing) guest/host MSR
1675	* pair to be swapped during the world-switch as part of the
1676	* auto-load/store MSR area in the VMCS.
1677	*
1678	* @returns VBox status code.
1679	* @param pVCpu The cross context virtual CPU structure.
1680	* @param pVmxTransient The VMX-transient structure.
1681	* @param idMsr The MSR.
1682	* @param uGuestMsrValue Value of the guest MSR.
1683	* @param fSetReadWrite Whether to set the guest read/write access of this
1684	* MSR (thus not causing a VM-exit).
1685	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1686	* necessary.
1687	*/
1688	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1689	bool fSetReadWrite, bool fUpdateHostMsr)
1690	{
1691	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1692	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1693	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1694	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1695	uint32_t i;
1696
1697	/* Paranoia. */
1698	Assert(pGuestMsrLoad);
1699
1700	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1701
1702	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1703	for (i = 0; i < cMsrs; i++)
1704	{
1705	if (pGuestMsrLoad[i].u32Msr == idMsr)
1706	break;
1707	}
1708
1709	bool fAdded = false;
1710	if (i == cMsrs)
1711	{
1712	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1713	++cMsrs;
1714	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1715	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1716
1717	/* Set the guest to read/write this MSR without causing VM-exits. */
1718	if ( fSetReadWrite
1719	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1720	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1721
1722	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1723	fAdded = true;
1724	}
1725
1726	/* Update the MSR value for the newly added or already existing MSR. */
1727	pGuestMsrLoad[i].u32Msr = idMsr;
1728	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1729
1730	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1731	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1732	{
1733	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1734	pGuestMsrStore[i].u32Msr = idMsr;
1735	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1736	}
1737
1738	/* Update the corresponding slot in the host MSR area. */
1739	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1740	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1741	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1742	pHostMsr[i].u32Msr = idMsr;
1743
1744	/*
1745	* Only if the caller requests to update the host MSR value AND we've newly added the
1746	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1747	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1748	*
1749	* We do this for performance reasons since reading MSRs may be quite expensive.
1750	*/
1751	if (fAdded)
1752	{
1753	if (fUpdateHostMsr)
1754	{
1755	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1756	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1757	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1758	}
1759	else
1760	{
1761	/* Someone else can do the work. */
1762	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1763	}
1764	}
1765	return VINF_SUCCESS;
1766	}
1767
1768
1769	/**
1770	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1771	* auto-load/store MSR area in the VMCS.
1772	*
1773	* @returns VBox status code.
1774	* @param pVCpu The cross context virtual CPU structure.
1775	* @param pVmxTransient The VMX-transient structure.
1776	* @param idMsr The MSR.
1777	*/
1778	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1779	{
1780	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1781	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1782	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1783	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1784
1785	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1786
1787	for (uint32_t i = 0; i < cMsrs; i++)
1788	{
1789	/* Find the MSR. */
1790	if (pGuestMsrLoad[i].u32Msr == idMsr)
1791	{
1792	/*
1793	* If it's the last MSR, we only need to reduce the MSR count.
1794	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1795	*/
1796	if (i < cMsrs - 1)
1797	{
1798	/* Remove it from the VM-entry MSR-load area. */
1799	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1800	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1801
1802	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1803	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1804	{
1805	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1806	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1807	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1808	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1809	}
1810
1811	/* Remove it from the VM-exit MSR-load area. */
1812	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1813	Assert(pHostMsr[i].u32Msr == idMsr);
1814	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1815	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1816	}
1817
1818	/* Reduce the count to reflect the removed MSR and bail. */
1819	--cMsrs;
1820	break;
1821	}
1822	}
1823
1824	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1825	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1826	{
1827	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1828	AssertRCReturn(rc, rc);
1829
1830	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1831	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1832	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1833
1834	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1835	return VINF_SUCCESS;
1836	}
1837
1838	return VERR_NOT_FOUND;
1839	}
1840
1841
1842	/**
1843	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1844	*
1845	* @returns @c true if found, @c false otherwise.
1846	* @param pVmcsInfo The VMCS info. object.
1847	* @param idMsr The MSR to find.
1848	*/
1849	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1850	{
1851	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1852	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1853	Assert(pMsrs);
1854	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1855	for (uint32_t i = 0; i < cMsrs; i++)
1856	{
1857	if (pMsrs[i].u32Msr == idMsr)
1858	return true;
1859	}
1860	return false;
1861	}
1862
1863
1864	/**
1865	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1866	*
1867	* @param pVCpu The cross context virtual CPU structure.
1868	* @param pVmcsInfo The VMCS info. object.
1869	*
1870	* @remarks No-long-jump zone!!!
1871	*/
1872	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1873	{
1874	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1875
1876	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1877	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1878	Assert(pHostMsrLoad);
1879	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1880	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1881	for (uint32_t i = 0; i < cMsrs; i++)
1882	{
1883	/*
1884	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1885	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1886	*/
1887	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1888	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1889	else
1890	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1891	}
1892	}
1893
1894
1895	/**
1896	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1897	* perform lazy restoration of the host MSRs while leaving VT-x.
1898	*
1899	* @param pVCpu The cross context virtual CPU structure.
1900	*
1901	* @remarks No-long-jump zone!!!
1902	*/
1903	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1904	{
1905	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1906
1907	/*
1908	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1909	*/
1910	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1911	{
1912	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1913	#if HC_ARCH_BITS == 64
1914	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1915	{
1916	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1917	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1918	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1919	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1920	}
1921	#endif
1922	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1923	}
1924	}
1925
1926
1927	/**
1928	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1929	* lazily while leaving VT-x.
1930	*
1931	* @returns true if it does, false otherwise.
1932	* @param pVCpu The cross context virtual CPU structure.
1933	* @param idMsr The MSR to check.
1934	*/
1935	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1936	{
1937	NOREF(pVCpu);
1938	#if HC_ARCH_BITS == 64
1939	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1940	{
1941	switch (idMsr)
1942	{
1943	case MSR_K8_LSTAR:
1944	case MSR_K6_STAR:
1945	case MSR_K8_SF_MASK:
1946	case MSR_K8_KERNEL_GS_BASE:
1947	return true;
1948	}
1949	}
1950	#else
1951	RT_NOREF(pVCpu, idMsr);
1952	#endif
1953	return false;
1954	}
1955
1956
1957	/**
1958	* Loads a set of guests MSRs to allow read/passthru to the guest.
1959	*
1960	* The name of this function is slightly confusing. This function does NOT
1961	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1962	* common prefix for functions dealing with "lazy restoration" of the shared
1963	* MSRs.
1964	*
1965	* @param pVCpu The cross context virtual CPU structure.
1966	*
1967	* @remarks No-long-jump zone!!!
1968	*/
1969	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1970	{
1971	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1972	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1973
1974	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1975	#if HC_ARCH_BITS == 64
1976	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1977	{
1978	/*
1979	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1980	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1981	* we can skip a few MSR writes.
1982	*
1983	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1984	* guest MSR values in the guest-CPU context might be different to what's currently
1985	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1986	* CPU, see @bugref{8728}.
1987	*/
1988	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1989	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1990	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1991	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1992	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1993	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1994	{
1995	#ifdef VBOX_STRICT
1996	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1997	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1998	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1999	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2000	#endif
2001	}
2002	else
2003	{
2004	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2005	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2006	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2007	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2008	}
2009	}
2010	#endif
2011	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2012	}
2013
2014
2015	/**
2016	* Performs lazy restoration of the set of host MSRs if they were previously
2017	* loaded with guest MSR values.
2018	*
2019	* @param pVCpu The cross context virtual CPU structure.
2020	*
2021	* @remarks No-long-jump zone!!!
2022	* @remarks The guest MSRs should have been saved back into the guest-CPU
2023	* context by hmR0VmxImportGuestState()!!!
2024	*/
2025	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2026	{
2027	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2028	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2029
2030	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2031	{
2032	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2033	#if HC_ARCH_BITS == 64
2034	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2035	{
2036	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2037	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2038	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2039	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2040	}
2041	#endif
2042	}
2043	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2044	}
2045
2046
2047	/**
2048	* Verifies that our cached values of the VMCS fields are all consistent with
2049	* what's actually present in the VMCS.
2050	*
2051	* @returns VBox status code.
2052	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2053	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2054	* VMCS content. HMCPU error-field is
2055	* updated, see VMX_VCI_XXX.
2056	* @param pVCpu The cross context virtual CPU structure.
2057	* @param pVmcsInfo The VMCS info. object.
2058	*/
2059	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2060	{
2061	uint32_t u32Val;
2062	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2063	AssertRCReturn(rc, rc);
2064	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2065	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2066	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2067	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2068
2069	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2070	AssertRCReturn(rc, rc);
2071	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2072	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2073	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2074	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2075
2076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2077	AssertRCReturn(rc, rc);
2078	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2079	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2080	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2081	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2082
2083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2084	AssertRCReturn(rc, rc);
2085	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2086	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2087	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2088	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2089
2090	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2091	{
2092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2093	AssertRCReturn(rc, rc);
2094	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2095	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2096	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2097	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2098	}
2099
2100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2101	AssertRCReturn(rc, rc);
2102	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2103	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2104	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2105	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2106
2107	uint64_t u64Val;
2108	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2109	AssertRCReturn(rc, rc);
2110	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2111	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2112	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2113	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2114
2115	return VINF_SUCCESS;
2116	}
2117
2118
2119	#ifdef VBOX_STRICT
2120	/**
2121	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2122	*
2123	* @param pVCpu The cross context virtual CPU structure.
2124	* @param pVmcsInfo The VMCS info. object.
2125	*/
2126	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2127	{
2128	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2129
2130	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2131	{
2132	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2133	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2134	uint64_t uVmcsEferMsrVmcs;
2135	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2136	AssertRC(rc);
2137
2138	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2139	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2140	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2141	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2142	}
2143	}
2144
2145
2146	/**
2147	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2148	* VMCS are correct.
2149	*
2150	* @param pVCpu The cross context virtual CPU structure.
2151	* @param pVmcsInfo The VMCS info. object.
2152	*/
2153	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2154	{
2155	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2156
2157	/* Read the various MSR-area counts from the VMCS. */
2158	uint32_t cEntryLoadMsrs;
2159	uint32_t cExitStoreMsrs;
2160	uint32_t cExitLoadMsrs;
2161	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2162	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2163	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2164
2165	/* Verify all the MSR counts are the same. */
2166	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2167	Assert(cExitStoreMsrs == cExitLoadMsrs);
2168	uint32_t const cMsrs = cExitLoadMsrs;
2169
2170	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2171	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2172
2173	/* Verify the MSR counts are within the allocated page size. */
2174	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2175
2176	/* Verify the relevant contents of the MSR areas match. */
2177	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2178	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2179	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2180	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2181	for (uint32_t i = 0; i < cMsrs; i++)
2182	{
2183	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2184	if (fSeparateExitMsrStorePage)
2185	{
2186	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2187	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2188	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2189	}
2190
2191	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2192	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2193	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2194
2195	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2196	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2197	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2198	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2199
2200	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2201	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2202	if (fIsEferMsr)
2203	{
2204	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2205	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2206	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2207	}
2208
2209	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2210	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2211	{
2212	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2213	if (fIsEferMsr)
2214	{
2215	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2216	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2217	}
2218	else
2219	{
2220	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2221	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2222	}
2223	}
2224
2225	/* Move to the next MSR. */
2226	pHostMsrLoad++;
2227	pGuestMsrLoad++;
2228	pGuestMsrStore++;
2229	}
2230	}
2231	#endif /* VBOX_STRICT */
2232
2233
2234	/**
2235	* Flushes the TLB using EPT.
2236	*
2237	* @returns VBox status code.
2238	* @param pVCpu The cross context virtual CPU structure of the calling
2239	* EMT. Can be NULL depending on @a enmTlbFlush.
2240	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2241	* enmTlbFlush.
2242	* @param enmTlbFlush Type of flush.
2243	*
2244	* @remarks Caller is responsible for making sure this function is called only
2245	* when NestedPaging is supported and providing @a enmTlbFlush that is
2246	* supported by the CPU.
2247	* @remarks Can be called with interrupts disabled.
2248	*/
2249	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2250	{
2251	uint64_t au64Descriptor[2];
2252	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2253	au64Descriptor[0] = 0;
2254	else
2255	{
2256	Assert(pVCpu);
2257	Assert(pVmcsInfo);
2258	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2259	}
2260	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2261
2262	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2263	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2264
2265	if ( RT_SUCCESS(rc)
2266	&& pVCpu)
2267	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2268	}
2269
2270
2271	/**
2272	* Flushes the TLB using VPID.
2273	*
2274	* @returns VBox status code.
2275	* @param pVCpu The cross context virtual CPU structure of the calling
2276	* EMT. Can be NULL depending on @a enmTlbFlush.
2277	* @param enmTlbFlush Type of flush.
2278	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2279	* on @a enmTlbFlush).
2280	*
2281	* @remarks Can be called with interrupts disabled.
2282	*/
2283	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2284	{
2285	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2286
2287	uint64_t au64Descriptor[2];
2288	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2289	{
2290	au64Descriptor[0] = 0;
2291	au64Descriptor[1] = 0;
2292	}
2293	else
2294	{
2295	AssertPtr(pVCpu);
2296	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2297	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2298	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2299	au64Descriptor[1] = GCPtr;
2300	}
2301
2302	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2303	AssertMsg(rc == VINF_SUCCESS,
2304	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2305
2306	if ( RT_SUCCESS(rc)
2307	&& pVCpu)
2308	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2309	NOREF(rc);
2310	}
2311
2312
2313	/**
2314	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2315	* otherwise there is nothing really to invalidate.
2316	*
2317	* @returns VBox status code.
2318	* @param pVCpu The cross context virtual CPU structure.
2319	* @param GCVirt Guest virtual address of the page to invalidate.
2320	*/
2321	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2322	{
2323	AssertPtr(pVCpu);
2324	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2325
2326	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2327	{
2328	/*
2329	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2330	* the EPT case. See @bugref{6043} and @bugref{6177}.
2331	*
2332	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2333	* as this function maybe called in a loop with individual addresses.
2334	*/
2335	PVM pVM = pVCpu->CTX_SUFF(pVM);
2336	if (pVM->hm.s.vmx.fVpid)
2337	{
2338	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2339
2340	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2341	/*
2342	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2343	* where executing INVVPID outside 64-bit mode does not flush translations of
2344	* 64-bit linear addresses, see @bugref{6208#c72}.
2345	*/
2346	if (RT_HI_U32(GCVirt))
2347	fVpidFlush = false;
2348	#endif
2349
2350	if (fVpidFlush)
2351	{
2352	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2353	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2354	}
2355	else
2356	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2357	}
2358	else if (pVM->hm.s.fNestedPaging)
2359	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2360	}
2361
2362	return VINF_SUCCESS;
2363	}
2364
2365
2366	/**
2367	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2368	* case where neither EPT nor VPID is supported by the CPU.
2369	*
2370	* @param pHostCpu The HM physical-CPU structure.
2371	* @param pVCpu The cross context virtual CPU structure.
2372	*
2373	* @remarks Called with interrupts disabled.
2374	*/
2375	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2376	{
2377	AssertPtr(pVCpu);
2378	AssertPtr(pHostCpu);
2379
2380	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2381
2382	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2383	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2384	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2385	pVCpu->hm.s.fForceTLBFlush = false;
2386	return;
2387	}
2388
2389
2390	/**
2391	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2392	*
2393	* @param pHostCpu The HM physical-CPU structure.
2394	* @param pVCpu The cross context virtual CPU structure.
2395	* @param pVmcsInfo The VMCS info. object.
2396	*
2397	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2398	* nomenclature. The reason is, to avoid confusion in compare statements
2399	* since the host-CPU copies are named "ASID".
2400	*
2401	* @remarks Called with interrupts disabled.
2402	*/
2403	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2404	{
2405	#ifdef VBOX_WITH_STATISTICS
2406	bool fTlbFlushed = false;
2407	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2408	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2409	if (!fTlbFlushed) \
2410	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2411	} while (0)
2412	#else
2413	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2414	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2415	#endif
2416
2417	AssertPtr(pVCpu);
2418	AssertPtr(pHostCpu);
2419	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2420
2421	PVM pVM = pVCpu->CTX_SUFF(pVM);
2422	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2423	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2424	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2425
2426	/*
2427	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2428	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2429	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2430	* cannot reuse the current ASID anymore.
2431	*/
2432	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2433	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2434	{
2435	++pHostCpu->uCurrentAsid;
2436	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2437	{
2438	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2439	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2440	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2441	}
2442
2443	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2444	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2445	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2446
2447	/*
2448	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2449	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2450	*/
2451	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2452	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2453	HMVMX_SET_TAGGED_TLB_FLUSHED();
2454	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2455	}
2456	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2457	{
2458	/*
2459	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2460	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2461	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2462	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2463	* mappings, see @bugref{6568}.
2464	*
2465	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2466	*/
2467	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2468	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2469	HMVMX_SET_TAGGED_TLB_FLUSHED();
2470	}
2471
2472	pVCpu->hm.s.fForceTLBFlush = false;
2473	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2474
2475	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2476	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2477	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2478	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2479	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2480	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2481	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2482	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2483	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2484
2485	/* Update VMCS with the VPID. */
2486	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2487	AssertRC(rc);
2488
2489	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2490	}
2491
2492
2493	/**
2494	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2495	*
2496	* @param pHostCpu The HM physical-CPU structure.
2497	* @param pVCpu The cross context virtual CPU structure.
2498	* @param pVmcsInfo The VMCS info. object.
2499	*
2500	* @remarks Called with interrupts disabled.
2501	*/
2502	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2503	{
2504	AssertPtr(pVCpu);
2505	AssertPtr(pHostCpu);
2506	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2507	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2508	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2509
2510	/*
2511	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2512	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2513	*/
2514	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2515	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2516	{
2517	pVCpu->hm.s.fForceTLBFlush = true;
2518	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2519	}
2520
2521	/* Check for explicit TLB flushes. */
2522	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2523	{
2524	pVCpu->hm.s.fForceTLBFlush = true;
2525	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2526	}
2527
2528	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2529	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2530
2531	if (pVCpu->hm.s.fForceTLBFlush)
2532	{
2533	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2534	pVCpu->hm.s.fForceTLBFlush = false;
2535	}
2536	}
2537
2538
2539	/**
2540	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2541	*
2542	* @param pHostCpu The HM physical-CPU structure.
2543	* @param pVCpu The cross context virtual CPU structure.
2544	*
2545	* @remarks Called with interrupts disabled.
2546	*/
2547	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2548	{
2549	AssertPtr(pVCpu);
2550	AssertPtr(pHostCpu);
2551	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2552	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2553	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2554
2555	/*
2556	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2557	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2558	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2559	* cannot reuse the current ASID anymore.
2560	*/
2561	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2562	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2563	{
2564	pVCpu->hm.s.fForceTLBFlush = true;
2565	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2566	}
2567
2568	/* Check for explicit TLB flushes. */
2569	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2570	{
2571	/*
2572	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2573	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2574	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2575	* include fExplicitFlush's too) - an obscure corner case.
2576	*/
2577	pVCpu->hm.s.fForceTLBFlush = true;
2578	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2579	}
2580
2581	PVM pVM = pVCpu->CTX_SUFF(pVM);
2582	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2583	if (pVCpu->hm.s.fForceTLBFlush)
2584	{
2585	++pHostCpu->uCurrentAsid;
2586	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2587	{
2588	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2589	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2590	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2591	}
2592
2593	pVCpu->hm.s.fForceTLBFlush = false;
2594	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2595	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2596	if (pHostCpu->fFlushAsidBeforeUse)
2597	{
2598	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2599	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2600	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2601	{
2602	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2603	pHostCpu->fFlushAsidBeforeUse = false;
2604	}
2605	else
2606	{
2607	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2608	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2609	}
2610	}
2611	}
2612
2613	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2614	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2615	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2616	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2617	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2618	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2619	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2620
2621	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2622	AssertRC(rc);
2623	}
2624
2625
2626	/**
2627	* Flushes the guest TLB entry based on CPU capabilities.
2628	*
2629	* @param pHostCpu The HM physical-CPU structure.
2630	* @param pVCpu The cross context virtual CPU structure.
2631	* @param pVmcsInfo The VMCS info. object.
2632	*
2633	* @remarks Called with interrupts disabled.
2634	*/
2635	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2636	{
2637	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2638	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2639	#endif
2640	PVM pVM = pVCpu->CTX_SUFF(pVM);
2641	switch (pVM->hm.s.vmx.enmTlbFlushType)
2642	{
2643	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2644	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2645	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2646	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2647	default:
2648	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2649	break;
2650	}
2651	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2652	}
2653
2654
2655	/**
2656	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2657	* TLB entries from the host TLB before VM-entry.
2658	*
2659	* @returns VBox status code.
2660	* @param pVM The cross context VM structure.
2661	*/
2662	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2663	{
2664	/*
2665	* Determine optimal flush type for nested paging.
2666	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2667	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2668	*/
2669	if (pVM->hm.s.fNestedPaging)
2670	{
2671	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2672	{
2673	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2674	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2675	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2676	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2677	else
2678	{
2679	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2680	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2681	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2682	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2683	}
2684
2685	/* Make sure the write-back cacheable memory type for EPT is supported. */
2686	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2687	{
2688	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2689	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2690	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2691	}
2692
2693	/* EPT requires a page-walk length of 4. */
2694	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2695	{
2696	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2697	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2698	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2699	}
2700	}
2701	else
2702	{
2703	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2704	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2705	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2706	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2707	}
2708	}
2709
2710	/*
2711	* Determine optimal flush type for VPID.
2712	*/
2713	if (pVM->hm.s.vmx.fVpid)
2714	{
2715	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2716	{
2717	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2718	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2719	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2720	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2721	else
2722	{
2723	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2724	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2725	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2726	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2727	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2728	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2729	pVM->hm.s.vmx.fVpid = false;
2730	}
2731	}
2732	else
2733	{
2734	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2735	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2736	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2737	pVM->hm.s.vmx.fVpid = false;
2738	}
2739	}
2740
2741	/*
2742	* Setup the handler for flushing tagged-TLBs.
2743	*/
2744	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2745	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2746	else if (pVM->hm.s.fNestedPaging)
2747	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2748	else if (pVM->hm.s.vmx.fVpid)
2749	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2750	else
2751	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2752	return VINF_SUCCESS;
2753	}
2754
2755
2756	/**
2757	* Sets up the virtual-APIC page address for the VMCS.
2758	*
2759	* @returns VBox status code.
2760	* @param pVCpu The cross context virtual CPU structure.
2761	* @param pVmcsInfo The VMCS info. object.
2762	*/
2763	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2764	{
2765	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2766	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2767	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2768	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2769	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2770	}
2771
2772
2773	/**
2774	* Sets up the MSR-bitmap address for the VMCS.
2775	*
2776	* @returns VBox status code.
2777	* @param pVCpu The cross context virtual CPU structure.
2778	* @param pVmcsInfo The VMCS info. object.
2779	*/
2780	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2781	{
2782	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2783	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2784	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2785	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2786	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2787	}
2788
2789
2790	/**
2791	* Sets up the APIC-access page address for the VMCS.
2792	*
2793	* @returns VBox status code.
2794	* @param pVCpu The cross context virtual CPU structure.
2795	*/
2796	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2797	{
2798	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2799	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2800	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2801	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2802	}
2803
2804
2805	/**
2806	* Sets up the VMCS link pointer for the VMCS.
2807	*
2808	* @returns VBox status code.
2809	* @param pVCpu The cross context virtual CPU structure.
2810	* @param pVmcsInfo The VMCS info. object.
2811	*/
2812	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2813	{
2814	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2815	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2816	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2817	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2818	}
2819
2820
2821	/**
2822	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2823	* in the VMCS.
2824	*
2825	* @returns VBox status code.
2826	* @param pVCpu The cross context virtual CPU structure.
2827	* @param pVmcsInfo The VMCS info. object.
2828	*/
2829	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2830	{
2831	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2832
2833	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2834	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2835	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2836
2837	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2838	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2839	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2840
2841	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2842	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2843	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2844
2845	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2846	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2847	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2848	AssertRCReturn(rc, rc);
2849	return VINF_SUCCESS;
2850	}
2851
2852
2853	/**
2854	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2855	*
2856	* @param pVCpu The cross context virtual CPU structure.
2857	* @param pVmcsInfo The VMCS info. object.
2858	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2859	*/
2860	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2861	{
2862	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2863
2864	/*
2865	* The guest can access the following MSRs (read, write) without causing
2866	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2867	*/
2868	PVM pVM = pVCpu->CTX_SUFF(pVM);
2869	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2871	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2872	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2873	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2874
2875	/*
2876	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2877	* associated with then. We never need to intercept access (writes need to be
2878	* executed without causing a VM-exit, reads will #GP fault anyway).
2879	*
2880	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2881	* read/write them. We swap the the guest/host MSR value using the
2882	* auto-load/store MSR area.
2883	*/
2884	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2885	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2886	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2887	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2888	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2889	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2890
2891	#if HC_ARCH_BITS == 64
2892	/*
2893	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2894	* required for 64-bit guests.
2895	*/
2896	if (pVM->hm.s.fAllow64BitGuests)
2897	{
2898	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2899	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2900	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2901	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2902	}
2903	#endif
2904
2905	/*
2906	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2907	*/
2908	#ifdef VBOX_STRICT
2909	Assert(pVmcsInfo->pvMsrBitmap);
2910	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2911	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2912	#endif
2913	}
2914
2915
2916	/**
2917	* Sets up pin-based VM-execution controls in the VMCS.
2918	*
2919	* @returns VBox status code.
2920	* @param pVCpu The cross context virtual CPU structure.
2921	* @param pVmcsInfo The VMCS info. object.
2922	*/
2923	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2924	{
2925	PVM pVM = pVCpu->CTX_SUFF(pVM);
2926	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2927	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2928
2929	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2930	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2931
2932	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2933	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2934
2935	/* Enable the VMX-preemption timer. */
2936	if (pVM->hm.s.vmx.fUsePreemptTimer)
2937	{
2938	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2939	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2940	}
2941
2942	#if 0
2943	/* Enable posted-interrupt processing. */
2944	if (pVM->hm.s.fPostedIntrs)
2945	{
2946	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2947	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2948	fVal \|= VMX_PIN_CTL_POSTED_INT;
2949	}
2950	#endif
2951
2952	if ((fVal & fZap) != fVal)
2953	{
2954	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2955	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2956	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2957	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2958	}
2959
2960	/* Commit it to the VMCS and update our cache. */
2961	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2962	AssertRCReturn(rc, rc);
2963	pVmcsInfo->u32PinCtls = fVal;
2964
2965	return VINF_SUCCESS;
2966	}
2967
2968
2969	/**
2970	* Sets up secondary processor-based VM-execution controls in the VMCS.
2971	*
2972	* @returns VBox status code.
2973	* @param pVCpu The cross context virtual CPU structure.
2974	* @param pVmcsInfo The VMCS info. object.
2975	*/
2976	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2977	{
2978	PVM pVM = pVCpu->CTX_SUFF(pVM);
2979	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2980	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2981
2982	/* WBINVD causes a VM-exit. */
2983	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2984	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2985
2986	/* Enable EPT (aka nested-paging). */
2987	if (pVM->hm.s.fNestedPaging)
2988	fVal \|= VMX_PROC_CTLS2_EPT;
2989
2990	/* Enable the INVPCID instruction if supported by the hardware and we expose
2991	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2992	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2993	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2994	fVal \|= VMX_PROC_CTLS2_INVPCID;
2995
2996	/* Enable VPID. */
2997	if (pVM->hm.s.vmx.fVpid)
2998	fVal \|= VMX_PROC_CTLS2_VPID;
2999
3000	/* Enable unrestricted guest execution. */
3001	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3002	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3003
3004	#if 0
3005	if (pVM->hm.s.fVirtApicRegs)
3006	{
3007	/* Enable APIC-register virtualization. */
3008	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3009	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3010
3011	/* Enable virtual-interrupt delivery. */
3012	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3013	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3014	}
3015	#endif
3016
3017	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3018	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3019	* done dynamically. */
3020	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3021	{
3022	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3023	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3024	AssertRCReturn(rc, rc);
3025	}
3026
3027	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3028	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3029	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3030	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3031	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3032
3033	/* Enable Pause-Loop exiting. */
3034	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3035	&& pVM->hm.s.vmx.cPleGapTicks
3036	&& pVM->hm.s.vmx.cPleWindowTicks)
3037	{
3038	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3039
3040	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3041	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3042	AssertRCReturn(rc, rc);
3043	}
3044
3045	if ((fVal & fZap) != fVal)
3046	{
3047	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3048	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3049	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3050	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3051	}
3052
3053	/* Commit it to the VMCS and update our cache. */
3054	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3055	AssertRCReturn(rc, rc);
3056	pVmcsInfo->u32ProcCtls2 = fVal;
3057
3058	return VINF_SUCCESS;
3059	}
3060
3061
3062	/**
3063	* Sets up processor-based VM-execution controls in the VMCS.
3064	*
3065	* @returns VBox status code.
3066	* @param pVCpu The cross context virtual CPU structure.
3067	* @param pVmcsInfo The VMCS info. object.
3068	*/
3069	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3070	{
3071	PVM pVM = pVCpu->CTX_SUFF(pVM);
3072
3073	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3074	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3075
3076	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3077	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3078	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3079	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3080	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3081	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3082	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3083
3084	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3085	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3086	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3087	{
3088	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3089	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3090	}
3091
3092	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3093	if (!pVM->hm.s.fNestedPaging)
3094	{
3095	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3096	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3097	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3098	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3099	}
3100
3101	/* Use TPR shadowing if supported by the CPU. */
3102	if ( PDMHasApic(pVM)
3103	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3104	{
3105	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3106	/* CR8 writes cause a VM-exit based on TPR threshold. */
3107	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3108	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3109	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3110	AssertRCReturn(rc, rc);
3111	}
3112	else
3113	{
3114	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3115	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3116	if (pVM->hm.s.fAllow64BitGuests)
3117	{
3118	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3119	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3120	}
3121	}
3122
3123	/* Use MSR-bitmaps if supported by the CPU. */
3124	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3125	{
3126	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3127	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3128	AssertRCReturn(rc, rc);
3129	}
3130
3131	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3132	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3133	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3134
3135	if ((fVal & fZap) != fVal)
3136	{
3137	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3138	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3139	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3140	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3141	}
3142
3143	/* Commit it to the VMCS and update our cache. */
3144	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3145	AssertRCReturn(rc, rc);
3146	pVmcsInfo->u32ProcCtls = fVal;
3147
3148	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3149	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3150	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3151
3152	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3153	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3154	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3155
3156	/* Sanity check, should not really happen. */
3157	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3158	{ /* likely */ }
3159	else
3160	{
3161	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3162	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3163	}
3164
3165	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3166	return VINF_SUCCESS;
3167	}
3168
3169
3170	/**
3171	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3172	* Processor-based VM-execution) control fields in the VMCS.
3173	*
3174	* @returns VBox status code.
3175	* @param pVCpu The cross context virtual CPU structure.
3176	* @param pVmcsInfo The VMCS info. object.
3177	*/
3178	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3179	{
3180	/* Set the auto-load/store MSR area addresses in the VMCS. */
3181	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3182	if (RT_SUCCESS(rc))
3183	{
3184	/* Set the VMCS link pointer in the VMCS. */
3185	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3186	if (RT_SUCCESS(rc))
3187	{
3188	/* Set the CR0/CR4 guest/host mask. */
3189	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3190	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3191	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3192	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3193	if (RT_SUCCESS(rc))
3194	{
3195	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3196	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3197	return VINF_SUCCESS;
3198	}
3199	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3200	}
3201	else
3202	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3203	}
3204	else
3205	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3206	return rc;
3207	}
3208
3209
3210	/**
3211	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3212	*
3213	* We shall setup those exception intercepts that don't change during the
3214	* lifetime of the VM here. The rest are done dynamically while loading the
3215	* guest state.
3216	*
3217	* @returns VBox status code.
3218	* @param pVCpu The cross context virtual CPU structure.
3219	* @param pVmcsInfo The VMCS info. object.
3220	*/
3221	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3222	{
3223	/*
3224	* The following exceptions are always intercepted:
3225	*
3226	* #AC - To prevent the guest from hanging the CPU.
3227	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3228	* recursive #DBs can cause a CPU hang.
3229	* #PF - To sync our shadow page tables when nested-paging is not used.
3230	*/
3231	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3232	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3233	\| RT_BIT(X86_XCPT_DB)
3234	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3235
3236	/* Commit it to the VMCS. */
3237	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3238	AssertRCReturn(rc, rc);
3239
3240	/* Update our cache of the exception bitmap. */
3241	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3242	return VINF_SUCCESS;
3243	}
3244
3245
3246	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3247	/**
3248	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3249	*
3250	* @returns VBox status code.
3251	* @param pVCpu The cross context virtual CPU structure.
3252	* @param pVmcsInfo The VMCS info. object.
3253	*/
3254	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3255	{
3256	PVM pVM = pVCpu->CTX_SUFF(pVM);
3257	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3258	if (RT_SUCCESS(rc))
3259	{
3260	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3261	if (RT_SUCCESS(rc))
3262	{
3263	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3264	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3265	if (RT_SUCCESS(rc))
3266	{
3267	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3268	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3269	if (RT_SUCCESS(rc))
3270	return VINF_SUCCESS;
3271
3272	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3273	}
3274	else
3275	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3276	}
3277	else
3278	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3279	}
3280	else
3281	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3282
3283	return rc;
3284	}
3285	#endif
3286
3287
3288	/**
3289	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3290	* VMX.
3291	*
3292	* @returns VBox status code.
3293	* @param pVCpu The cross context virtual CPU structure.
3294	* @param pVmcsInfo The VMCS info. object.
3295	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3296	*/
3297	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3298	{
3299	Assert(pVmcsInfo);
3300	Assert(pVmcsInfo->pvVmcs);
3301	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3302
3303	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3304	PVM pVM = pVCpu->CTX_SUFF(pVM);
3305	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3306	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3307
3308	LogFlowFunc(("\n"));
3309
3310	/*
3311	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3312	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3313	*/
3314	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3315	if (RT_SUCCESS(rc))
3316	{
3317	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3318	if (RT_SUCCESS(rc))
3319	{
3320	if (!fIsNstGstVmcs)
3321	{
3322	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3323	if (RT_SUCCESS(rc))
3324	{
3325	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3326	if (RT_SUCCESS(rc))
3327	{
3328	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3329	if (RT_SUCCESS(rc))
3330	{
3331	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3332	if (RT_SUCCESS(rc))
3333	{ /* likely */ }
3334	else
3335	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3336	}
3337	else
3338	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3339	}
3340	else
3341	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3342	}
3343	else
3344	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3345	}
3346	else
3347	{
3348	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3349	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3350	if (RT_SUCCESS(rc))
3351	{ /* likely */ }
3352	else
3353	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3354	#else
3355	AssertFailed();
3356	#endif
3357	}
3358	}
3359	else
3360	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3361	}
3362	else
3363	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3364
3365	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3366	if (RT_SUCCESS(rc))
3367	{
3368	rc = hmR0VmxClearVmcs(pVmcsInfo);
3369	if (RT_SUCCESS(rc))
3370	{ /* likely */ }
3371	else
3372	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3373	}
3374
3375	/*
3376	* Update the last-error record both for failures and success, so we
3377	* can propagate the status code back to ring-3 for diagnostics.
3378	*/
3379	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3380	NOREF(pszVmcs);
3381	return rc;
3382	}
3383
3384
3385	/**
3386	* Does global VT-x initialization (called during module initialization).
3387	*
3388	* @returns VBox status code.
3389	*/
3390	VMMR0DECL(int) VMXR0GlobalInit(void)
3391	{
3392	#ifdef HMVMX_USE_FUNCTION_TABLE
3393	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3394	# ifdef VBOX_STRICT
3395	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3396	Assert(g_apfnVMExitHandlers[i]);
3397	# endif
3398	#endif
3399	return VINF_SUCCESS;
3400	}
3401
3402
3403	/**
3404	* Does global VT-x termination (called during module termination).
3405	*/
3406	VMMR0DECL(void) VMXR0GlobalTerm()
3407	{
3408	/* Nothing to do currently. */
3409	}
3410
3411
3412	/**
3413	* Sets up and activates VT-x on the current CPU.
3414	*
3415	* @returns VBox status code.
3416	* @param pHostCpu The HM physical-CPU structure.
3417	* @param pVM The cross context VM structure. Can be
3418	* NULL after a host resume operation.
3419	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3420	* fEnabledByHost is @c true).
3421	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3422	* @a fEnabledByHost is @c true).
3423	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3424	* enable VT-x on the host.
3425	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3426	*/
3427	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3428	PCSUPHWVIRTMSRS pHwvirtMsrs)
3429	{
3430	Assert(pHostCpu);
3431	Assert(pHwvirtMsrs);
3432	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3433
3434	/* Enable VT-x if it's not already enabled by the host. */
3435	if (!fEnabledByHost)
3436	{
3437	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3438	if (RT_FAILURE(rc))
3439	return rc;
3440	}
3441
3442	/*
3443	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3444	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3445	* invalidated when flushing by VPID.
3446	*/
3447	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3448	{
3449	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3450	pHostCpu->fFlushAsidBeforeUse = false;
3451	}
3452	else
3453	pHostCpu->fFlushAsidBeforeUse = true;
3454
3455	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3456	++pHostCpu->cTlbFlushes;
3457
3458	return VINF_SUCCESS;
3459	}
3460
3461
3462	/**
3463	* Deactivates VT-x on the current CPU.
3464	*
3465	* @returns VBox status code.
3466	* @param pvCpuPage Pointer to the VMXON region.
3467	* @param HCPhysCpuPage Physical address of the VMXON region.
3468	*
3469	* @remarks This function should never be called when SUPR0EnableVTx() or
3470	* similar was used to enable VT-x on the host.
3471	*/
3472	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3473	{
3474	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3475
3476	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3477	return hmR0VmxLeaveRootMode();
3478	}
3479
3480
3481	/**
3482	* Does per-VM VT-x initialization.
3483	*
3484	* @returns VBox status code.
3485	* @param pVM The cross context VM structure.
3486	*/
3487	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3488	{
3489	LogFlowFunc(("pVM=%p\n", pVM));
3490
3491	int rc = hmR0VmxStructsAlloc(pVM);
3492	if (RT_FAILURE(rc))
3493	{
3494	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3495	return rc;
3496	}
3497
3498	return VINF_SUCCESS;
3499	}
3500
3501
3502	/**
3503	* Does per-VM VT-x termination.
3504	*
3505	* @returns VBox status code.
3506	* @param pVM The cross context VM structure.
3507	*/
3508	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3509	{
3510	LogFlowFunc(("pVM=%p\n", pVM));
3511
3512	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3513	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3514	{
3515	Assert(pVM->hm.s.vmx.pvScratch);
3516	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3517	}
3518	#endif
3519	hmR0VmxStructsFree(pVM);
3520	return VINF_SUCCESS;
3521	}
3522
3523
3524	/**
3525	* Sets up the VM for execution using hardware-assisted VMX.
3526	* This function is only called once per-VM during initialization.
3527	*
3528	* @returns VBox status code.
3529	* @param pVM The cross context VM structure.
3530	*/
3531	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3532	{
3533	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3534	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3535
3536	LogFlowFunc(("pVM=%p\n", pVM));
3537
3538	/*
3539	* At least verify if VMX is enabled, since we can't check if we're in
3540	* VMX root mode or not without causing a #GP.
3541	*/
3542	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3543	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3544	{ /* likely */ }
3545	else
3546	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3547
3548	/*
3549	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3550	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3551	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3552	*/
3553	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3554	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3555	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3556	{
3557	LogRelFunc(("Invalid real-on-v86 state.\n"));
3558	return VERR_INTERNAL_ERROR;
3559	}
3560
3561	/* Initialize these always, see hmR3InitFinalizeR0().*/
3562	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3563	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3564
3565	/* Setup the tagged-TLB flush handlers. */
3566	int rc = hmR0VmxSetupTaggedTlb(pVM);
3567	if (RT_FAILURE(rc))
3568	{
3569	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3570	return rc;
3571	}
3572
3573	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3574	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3575	#if HC_ARCH_BITS == 64
3576	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3577	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3578	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3579	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3580	#endif
3581
3582	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3583	{
3584	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3585	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3586
3587	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3588	if (RT_SUCCESS(rc))
3589	{
3590	#if HC_ARCH_BITS == 32
3591	hmR0VmxInitVmcsReadCache(pVCpu);
3592	#endif
3593	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3594	if (pVM->cpum.ro.GuestFeatures.fVmx)
3595	{
3596	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3597	if (RT_SUCCESS(rc))
3598	{ /* likely */ }
3599	else
3600	{
3601	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3602	return rc;
3603	}
3604	}
3605	#endif
3606	}
3607	else
3608	{
3609	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3610	return rc;
3611	}
3612	}
3613
3614	return VINF_SUCCESS;
3615	}
3616
3617
3618	#if HC_ARCH_BITS == 32
3619	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3620	/**
3621	* Check if guest state allows safe use of 32-bit switcher again.
3622	*
3623	* Segment bases and protected mode structures must be 32-bit addressable
3624	* because the 32-bit switcher will ignore high dword when writing these VMCS
3625	* fields. See @bugref{8432} for details.
3626	*
3627	* @returns true if safe, false if must continue to use the 64-bit switcher.
3628	* @param pCtx Pointer to the guest-CPU context.
3629	*
3630	* @remarks No-long-jump zone!!!
3631	*/
3632	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3633	{
3634	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3635	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3636	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3637	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3638	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3639	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3640	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3641	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3642	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3643	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3644
3645	/* All good, bases are 32-bit. */
3646	return true;
3647	}
3648	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3649
3650	# ifdef VBOX_STRICT
3651	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3652	{
3653	switch (idxField)
3654	{
3655	case VMX_VMCS_GUEST_RIP:
3656	case VMX_VMCS_GUEST_RSP:
3657	case VMX_VMCS_GUEST_SYSENTER_EIP:
3658	case VMX_VMCS_GUEST_SYSENTER_ESP:
3659	case VMX_VMCS_GUEST_GDTR_BASE:
3660	case VMX_VMCS_GUEST_IDTR_BASE:
3661	case VMX_VMCS_GUEST_CS_BASE:
3662	case VMX_VMCS_GUEST_DS_BASE:
3663	case VMX_VMCS_GUEST_ES_BASE:
3664	case VMX_VMCS_GUEST_FS_BASE:
3665	case VMX_VMCS_GUEST_GS_BASE:
3666	case VMX_VMCS_GUEST_SS_BASE:
3667	case VMX_VMCS_GUEST_LDTR_BASE:
3668	case VMX_VMCS_GUEST_TR_BASE:
3669	case VMX_VMCS_GUEST_CR3:
3670	return true;
3671	}
3672	return false;
3673	}
3674
3675	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3676	{
3677	switch (idxField)
3678	{
3679	/* Read-only fields. */
3680	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3681	return true;
3682	}
3683	/* Remaining readable fields should also be writable. */
3684	return hmR0VmxIsValidWriteField(idxField);
3685	}
3686	# endif /* VBOX_STRICT */
3687
3688
3689	/**
3690	* Executes the specified handler in 64-bit mode.
3691	*
3692	* @returns VBox status code (no informational status codes).
3693	* @param pVCpu The cross context virtual CPU structure.
3694	* @param enmOp The operation to perform.
3695	* @param cParams Number of parameters.
3696	* @param paParam Array of 32-bit parameters.
3697	*/
3698	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3699	{
3700	PVM pVM = pVCpu->CTX_SUFF(pVM);
3701	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3702	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3703	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3704	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3705
3706	#ifdef VBOX_STRICT
3707	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3708	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3709
3710	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3711	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3712	#endif
3713
3714	/* Disable interrupts. */
3715	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3716
3717	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3718	RTCPUID idHostCpu = RTMpCpuId();
3719	CPUMR0SetLApic(pVCpu, idHostCpu);
3720	#endif
3721
3722	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3723
3724	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3725	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3726
3727	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3728	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3729	hmR0VmxClearVmcs(pVmcsInfo);
3730
3731	/* Leave VMX root mode and disable VMX. */
3732	VMXDisable();
3733	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3734
3735	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3736	CPUMSetHyperEIP(pVCpu, enmOp);
3737	for (int i = (int)cParams - 1; i >= 0; i--)
3738	CPUMPushHyper(pVCpu, paParam[i]);
3739
3740	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3741
3742	/* Call the switcher. */
3743	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3744	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3745
3746	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3747	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3748
3749	/* Re-enter VMX root mode. */
3750	int rc2 = VMXEnable(HCPhysCpuPage);
3751	if (RT_FAILURE(rc2))
3752	{
3753	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3754	ASMSetFlags(fOldEFlags);
3755	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3756	return rc2;
3757	}
3758
3759	/* Restore the VMCS as the current VMCS. */
3760	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3761	AssertRC(rc2);
3762	Assert(!(ASMGetFlags() & X86_EFL_IF));
3763	ASMSetFlags(fOldEFlags);
3764	return rc;
3765	}
3766
3767
3768	/**
3769	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3770	* supporting 64-bit guests.
3771	*
3772	* @returns VBox status code.
3773	* @param fResume Whether to VMLAUNCH or VMRESUME.
3774	* @param pCtx Pointer to the guest-CPU context.
3775	* @param pCache Pointer to the VMCS batch cache.
3776	* @param pVM The cross context VM structure.
3777	* @param pVCpu The cross context virtual CPU structure.
3778	*/
3779	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3780	{
3781	NOREF(fResume);
3782
3783	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3784	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3785	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3786
3787	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3788	pCache->uPos = 1;
3789	pCache->interPD = PGMGetInterPaeCR3(pVM);
3790	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3791	#endif
3792
3793	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3794	pCache->TestIn.HCPhysCpuPage = 0;
3795	pCache->TestIn.HCPhysVmcs = 0;
3796	pCache->TestIn.pCache = 0;
3797	pCache->TestOut.HCPhysVmcs = 0;
3798	pCache->TestOut.pCache = 0;
3799	pCache->TestOut.pCtx = 0;
3800	pCache->TestOut.eflags = 0;
3801	#else
3802	NOREF(pCache);
3803	#endif
3804
3805	uint32_t aParam[10];
3806	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3807	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3808	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3809	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3810	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3811	aParam[5] = 0;
3812	aParam[6] = VM_RC_ADDR(pVM, pVM);
3813	aParam[7] = 0;
3814	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3815	aParam[9] = 0;
3816
3817	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3818	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3819	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3820	#endif
3821	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3822
3823	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3824	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3825	Assert(pCtx->dr[4] == 10);
3826	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3827	#endif
3828
3829	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3830	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3831	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3832	pVmcsInfo->HCPhysVmcs));
3833	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3834	pCache->TestOut.HCPhysVmcs));
3835	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3836	pCache->TestOut.pCache));
3837	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3838	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3839	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3840	pCache->TestOut.pCtx));
3841	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3842	#endif
3843	NOREF(pCtx);
3844	return rc;
3845	}
3846	#endif
3847
3848
3849	/**
3850	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3851	* the VMCS.
3852	*
3853	* @returns VBox status code.
3854	*/
3855	static int hmR0VmxExportHostControlRegs(void)
3856	{
3857	RTCCUINTREG uReg = ASMGetCR0();
3858	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3859	AssertRCReturn(rc, rc);
3860
3861	uReg = ASMGetCR3();
3862	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3863	AssertRCReturn(rc, rc);
3864
3865	uReg = ASMGetCR4();
3866	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3867	AssertRCReturn(rc, rc);
3868	return rc;
3869	}
3870
3871
3872	/**
3873	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3874	* the host-state area in the VMCS.
3875	*
3876	* @returns VBox status code.
3877	* @param pVCpu The cross context virtual CPU structure.
3878	*/
3879	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3880	{
3881	#if HC_ARCH_BITS == 64
3882	/**
3883	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3884	* requirements. See hmR0VmxExportHostSegmentRegs().
3885	*/
3886	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3887	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3888	{ \
3889	bool fValidSelector = true; \
3890	if ((selValue) & X86_SEL_LDT) \
3891	{ \
3892	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3893	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3894	} \
3895	if (fValidSelector) \
3896	{ \
3897	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3898	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3899	} \
3900	(selValue) = 0; \
3901	}
3902
3903	/*
3904	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3905	* will be messed up. We should -not- save the messed up state without restoring
3906	* the original host-state, see @bugref{7240}.
3907	*
3908	* This apparently can happen (most likely the FPU changes), deal with it rather than
3909	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3910	*/
3911	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3912	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3913	{
3914	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3915	pVCpu->idCpu));
3916	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3917	}
3918	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3919	#else
3920	RT_NOREF(pVCpu);
3921	#endif
3922
3923	/*
3924	* Host DS, ES, FS and GS segment registers.
3925	*/
3926	#if HC_ARCH_BITS == 64
3927	RTSEL uSelDS = ASMGetDS();
3928	RTSEL uSelES = ASMGetES();
3929	RTSEL uSelFS = ASMGetFS();
3930	RTSEL uSelGS = ASMGetGS();
3931	#else
3932	RTSEL uSelDS = 0;
3933	RTSEL uSelES = 0;
3934	RTSEL uSelFS = 0;
3935	RTSEL uSelGS = 0;
3936	#endif
3937
3938	/*
3939	* Host CS and SS segment registers.
3940	*/
3941	RTSEL uSelCS = ASMGetCS();
3942	RTSEL uSelSS = ASMGetSS();
3943
3944	/*
3945	* Host TR segment register.
3946	*/
3947	RTSEL uSelTR = ASMGetTR();
3948
3949	#if HC_ARCH_BITS == 64
3950	/*
3951	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3952	* gain VM-entry and restore them before we get preempted.
3953	*
3954	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3955	*/
3956	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3957	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3958	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3959	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3960	# undef VMXLOCAL_ADJUST_HOST_SEG
3961	#endif
3962
3963	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3964	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3965	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3966	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3967	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3968	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3969	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3970	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3971	Assert(uSelCS);
3972	Assert(uSelTR);
3973
3974	/* Write these host selector fields into the host-state area in the VMCS. */
3975	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3976	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3977	#if HC_ARCH_BITS == 64
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3979	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3980	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3981	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3982	#else
3983	NOREF(uSelDS);
3984	NOREF(uSelES);
3985	NOREF(uSelFS);
3986	NOREF(uSelGS);
3987	#endif
3988	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3989	AssertRCReturn(rc, rc);
3990
3991	/*
3992	* Host GDTR and IDTR.
3993	*/
3994	RTGDTR Gdtr;
3995	RTIDTR Idtr;
3996	RT_ZERO(Gdtr);
3997	RT_ZERO(Idtr);
3998	ASMGetGDTR(&Gdtr);
3999	ASMGetIDTR(&Idtr);
4000	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4001	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4002	AssertRCReturn(rc, rc);
4003
4004	#if HC_ARCH_BITS == 64
4005	/*
4006	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4007	* them to the maximum limit (0xffff) on every VM-exit.
4008	*/
4009	if (Gdtr.cbGdt != 0xffff)
4010	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4011
4012	/*
4013	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4014	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4015	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4016	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4017	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4018	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4019	* at 0xffff on hosts where we are sure it won't cause trouble.
4020	*/
4021	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4022	if (Idtr.cbIdt < 0x0fff)
4023	# else
4024	if (Idtr.cbIdt != 0xffff)
4025	# endif
4026	{
4027	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4028	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4029	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4030	}
4031	#endif
4032
4033	/*
4034	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4035	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4036	* RPL should be too in most cases.
4037	*/
4038	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4039	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4040
4041	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4042	#if HC_ARCH_BITS == 64
4043	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4044
4045	/*
4046	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4047	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4048	* restoration if the host has something else. Task switching is not supported in 64-bit
4049	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4050	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4051	*
4052	* [1] See Intel spec. 3.5 "System Descriptor Types".
4053	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4054	*/
4055	PVM pVM = pVCpu->CTX_SUFF(pVM);
4056	Assert(pDesc->System.u4Type == 11);
4057	if ( pDesc->System.u16LimitLow != 0x67
4058	\|\| pDesc->System.u4LimitHigh)
4059	{
4060	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4061	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4062	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4063	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4064	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4065	}
4066
4067	/*
4068	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4069	*/
4070	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4071	{
4072	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4073	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4074	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4075	{
4076	/* The GDT is read-only but the writable GDT is available. */
4077	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4078	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4079	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4080	AssertRCReturn(rc, rc);
4081	}
4082	}
4083	#else
4084	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4085	#endif
4086	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4087	AssertRCReturn(rc, rc);
4088
4089	/*
4090	* Host FS base and GS base.
4091	*/
4092	#if HC_ARCH_BITS == 64
4093	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4094	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4095	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4096	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4097	AssertRCReturn(rc, rc);
4098
4099	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4100	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4101	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4102	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4103	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4104	#endif
4105	return VINF_SUCCESS;
4106	}
4107
4108
4109	/**
4110	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4111	* host-state area of the VMCS.
4112	*
4113	* These MSRs will be automatically restored on the host after every successful
4114	* VM-exit.
4115	*
4116	* @returns VBox status code.
4117	* @param pVCpu The cross context virtual CPU structure.
4118	*
4119	* @remarks No-long-jump zone!!!
4120	*/
4121	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4122	{
4123	AssertPtr(pVCpu);
4124
4125	/*
4126	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4127	* rather than swapping them on every VM-entry.
4128	*/
4129	hmR0VmxLazySaveHostMsrs(pVCpu);
4130
4131	/*
4132	* Host Sysenter MSRs.
4133	*/
4134	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4135	#if HC_ARCH_BITS == 32
4136	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4137	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4138	#else
4139	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4140	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4141	#endif
4142	AssertRCReturn(rc, rc);
4143
4144	/*
4145	* Host EFER MSR.
4146	*
4147	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4148	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4149	*/
4150	PVM pVM = pVCpu->CTX_SUFF(pVM);
4151	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4152	{
4153	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4154	AssertRCReturn(rc, rc);
4155	}
4156
4157	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4158	* hmR0VmxExportGuestEntryExitCtls(). */
4159
4160	return VINF_SUCCESS;
4161	}
4162
4163
4164	/**
4165	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4166	*
4167	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4168	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4169	*
4170	* @returns true if we need to load guest EFER, false otherwise.
4171	* @param pVCpu The cross context virtual CPU structure.
4172	*
4173	* @remarks Requires EFER, CR4.
4174	* @remarks No-long-jump zone!!!
4175	*/
4176	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4177	{
4178	#ifdef HMVMX_ALWAYS_SWAP_EFER
4179	RT_NOREF(pVCpu);
4180	return true;
4181	#else
4182	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4183	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4184	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4185	if (CPUMIsGuestInLongModeEx(pCtx))
4186	return false;
4187	#endif
4188
4189	PVM pVM = pVCpu->CTX_SUFF(pVM);
4190	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4191	uint64_t const u64GuestEfer = pCtx->msrEFER;
4192
4193	/*
4194	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4195	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4196	*/
4197	if ( CPUMIsGuestInLongModeEx(pCtx)
4198	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4199	return true;
4200
4201	/*
4202	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4203	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4204	*
4205	* See Intel spec. 4.5 "IA-32e Paging".
4206	* See Intel spec. 4.1.1 "Three Paging Modes".
4207	*
4208	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4209	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4210	*/
4211	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4212	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4213	if ( (pCtx->cr4 & X86_CR4_PAE)
4214	&& (pCtx->cr0 & X86_CR0_PG)
4215	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4216	{
4217	/* Assert that host is NX capable. */
4218	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4219	return true;
4220	}
4221
4222	return false;
4223	#endif
4224	}
4225
4226	/**
4227	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4228	* VMCS.
4229	*
4230	* This is typically required when the guest changes paging mode.
4231	*
4232	* @returns VBox status code.
4233	* @param pVCpu The cross context virtual CPU structure.
4234	* @param pVmxTransient The VMX-transient structure.
4235	*
4236	* @remarks Requires EFER.
4237	* @remarks No-long-jump zone!!!
4238	*/
4239	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4240	{
4241	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4242	{
4243	PVM pVM = pVCpu->CTX_SUFF(pVM);
4244	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4245
4246	/*
4247	* VM-entry controls.
4248	*/
4249	{
4250	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4251	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4252
4253	/*
4254	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4255	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4256	*
4257	* For nested-guests, this is a mandatory VM-entry control. It's also
4258	* required because we do not want to leak host bits to the nested-guest.
4259	*/
4260	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4261
4262	/*
4263	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4264	*
4265	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4266	* required to get the nested-guest working with hardware-assisted VMX execution.
4267	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4268	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4269	* here rather than while merging the guest VMCS controls.
4270	*/
4271	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4272	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4273	else
4274	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4275
4276	/*
4277	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4278	*
4279	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4280	* regardless of whether the nested-guest VMCS specifies it because we are free to
4281	* load whatever MSRs we require and we do not need to modify the guest visible copy
4282	* of the VM-entry MSR load area.
4283	*/
4284	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4285	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4286	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4287	else
4288	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4289
4290	/*
4291	* The following should -not- be set (since we're not in SMM mode):
4292	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4293	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4294	*/
4295
4296	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4297	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4298
4299	if ((fVal & fZap) == fVal)
4300	{ /* likely */ }
4301	else
4302	{
4303	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4304	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4305	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4306	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4307	}
4308
4309	/* Commit it to the VMCS. */
4310	if (pVmcsInfo->u32EntryCtls != fVal)
4311	{
4312	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4313	AssertRCReturn(rc, rc);
4314	pVmcsInfo->u32EntryCtls = fVal;
4315	}
4316	}
4317
4318	/*
4319	* VM-exit controls.
4320	*/
4321	{
4322	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4323	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4324
4325	/*
4326	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4327	* supported the 1-setting of this bit.
4328	*
4329	* For nested-guests, we set the "save debug controls" as the converse
4330	* "load debug controls" is mandatory for nested-guests anyway.
4331	*/
4332	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4333
4334	/*
4335	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4336	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4337	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4338	* hmR0VmxExportHostMsrs().
4339	*
4340	* For nested-guests, we always set this bit as we do not support 32-bit
4341	* hosts.
4342	*/
4343	#if HC_ARCH_BITS == 64
4344	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4345	#else
4346	Assert(!pVmxTransient->fIsNestedGuest);
4347	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4348	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4349	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4350	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4351	{
4352	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4353	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4354	}
4355	else
4356	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4357	#endif
4358
4359	/*
4360	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4361	*
4362	* For nested-guests, we should use the "save IA32_EFER" control if we also
4363	* used the "load IA32_EFER" control while exporting VM-entry controls.
4364	*/
4365	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4366	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4367	{
4368	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4369	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4370	}
4371
4372	/*
4373	* Enable saving of the VMX-preemption timer value on VM-exit.
4374	* For nested-guests, currently not exposed/used.
4375	*/
4376	if ( pVM->hm.s.vmx.fUsePreemptTimer
4377	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4378	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4379
4380	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4381	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4382
4383	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4384	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4385	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4386
4387	if ((fVal & fZap) == fVal)
4388	{ /* likely */ }
4389	else
4390	{
4391	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4392	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4393	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4394	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4395	}
4396
4397	/* Commit it to the VMCS. */
4398	if (pVmcsInfo->u32ExitCtls != fVal)
4399	{
4400	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4401	AssertRCReturn(rc, rc);
4402	pVmcsInfo->u32ExitCtls = fVal;
4403	}
4404	}
4405
4406	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4407	}
4408	return VINF_SUCCESS;
4409	}
4410
4411
4412	/**
4413	* Sets the TPR threshold in the VMCS.
4414	*
4415	* @returns VBox status code.
4416	* @param pVCpu The cross context virtual CPU structure.
4417	* @param pVmcsInfo The VMCS info. object.
4418	* @param u32TprThreshold The TPR threshold (task-priority class only).
4419	*/
4420	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4421	{
4422	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4423	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4424	RT_NOREF2(pVCpu, pVmcsInfo);
4425	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4426	}
4427
4428
4429	/**
4430	* Exports the guest APIC TPR state into the VMCS.
4431	*
4432	* @returns VBox status code.
4433	* @param pVCpu The cross context virtual CPU structure.
4434	* @param pVmxTransient The VMX-transient structure.
4435	*
4436	* @remarks No-long-jump zone!!!
4437	*/
4438	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4439	{
4440	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4441	{
4442	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4443
4444	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4445	if (!pVmxTransient->fIsNestedGuest)
4446	{
4447	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4448	&& APICIsEnabled(pVCpu))
4449	{
4450	/*
4451	* Setup TPR shadowing.
4452	*/
4453	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4454	{
4455	bool fPendingIntr = false;
4456	uint8_t u8Tpr = 0;
4457	uint8_t u8PendingIntr = 0;
4458	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4459	AssertRCReturn(rc, rc);
4460
4461	/*
4462	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4463	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4464	* priority of the pending interrupt so we can deliver the interrupt. If there
4465	* are no interrupts pending, set threshold to 0 to not cause any
4466	* TPR-below-threshold VM-exits.
4467	*/
4468	Assert(pVmcsInfo->pbVirtApic);
4469	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4470	uint32_t u32TprThreshold = 0;
4471	if (fPendingIntr)
4472	{
4473	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4474	(which is the Task-Priority Class). */
4475	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4476	const uint8_t u8TprPriority = u8Tpr >> 4;
4477	if (u8PendingPriority <= u8TprPriority)
4478	u32TprThreshold = u8PendingPriority;
4479	}
4480
4481	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4482	AssertRCReturn(rc, rc);
4483	}
4484	}
4485	}
4486	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4487	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4488	}
4489	return VINF_SUCCESS;
4490	}
4491
4492
4493	/**
4494	* Gets the guest interruptibility-state.
4495	*
4496	* @returns Guest's interruptibility-state.
4497	* @param pVCpu The cross context virtual CPU structure.
4498	* @param pVmcsInfo The VMCS info. object.
4499	*
4500	* @remarks No-long-jump zone!!!
4501	*/
4502	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4503	{
4504	/*
4505	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4506	*/
4507	uint32_t fIntrState = 0;
4508	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4509	{
4510	/* If inhibition is active, RIP and RFLAGS should've been updated
4511	(i.e. read previously from the VMCS or from ring-3). */
4512	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4513	#ifdef VBOX_STRICT
4514	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4515	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4516	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4517	#endif
4518	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4519	{
4520	if (pCtx->eflags.Bits.u1IF)
4521	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4522	else
4523	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4524	}
4525	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4526	{
4527	/*
4528	* We can clear the inhibit force flag as even if we go back to the recompiler
4529	* without executing guest code in VT-x, the flag's condition to be cleared is
4530	* met and thus the cleared state is correct.
4531	*/
4532	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4533	}
4534	}
4535
4536	/*
4537	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4538	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4539	* setting this would block host-NMIs and IRET will not clear the blocking.
4540	*
4541	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4542	*
4543	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4544	*/
4545	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4546	&& CPUMIsGuestNmiBlocking(pVCpu))
4547	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4548
4549	return fIntrState;
4550	}
4551
4552
4553	/**
4554	* Exports the exception intercepts required for guest execution in the VMCS.
4555	*
4556	* @returns VBox status code.
4557	* @param pVCpu The cross context virtual CPU structure.
4558	* @param pVmxTransient The VMX-transient structure.
4559	*
4560	* @remarks No-long-jump zone!!!
4561	*/
4562	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4563	{
4564	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4565	{
4566	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4567	if ( !pVmxTransient->fIsNestedGuest
4568	&& pVCpu->hm.s.fGIMTrapXcptUD)
4569	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4570	else
4571	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4572
4573	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4574	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4575	}
4576	return VINF_SUCCESS;
4577	}
4578
4579
4580	/**
4581	* Exports the guest's RIP into the guest-state area in the VMCS.
4582	*
4583	* @returns VBox status code.
4584	* @param pVCpu The cross context virtual CPU structure.
4585	*
4586	* @remarks No-long-jump zone!!!
4587	*/
4588	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4589	{
4590	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4591	{
4592	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4593
4594	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4595	AssertRCReturn(rc, rc);
4596
4597	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4598	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4599	}
4600	return VINF_SUCCESS;
4601	}
4602
4603
4604	/**
4605	* Exports the guest's RSP into the guest-state area in the VMCS.
4606	*
4607	* @returns VBox status code.
4608	* @param pVCpu The cross context virtual CPU structure.
4609	*
4610	* @remarks No-long-jump zone!!!
4611	*/
4612	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4613	{
4614	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4615	{
4616	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4617
4618	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4619	AssertRCReturn(rc, rc);
4620
4621	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4622	}
4623	return VINF_SUCCESS;
4624	}
4625
4626
4627	/**
4628	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4629	*
4630	* @returns VBox status code.
4631	* @param pVCpu The cross context virtual CPU structure.
4632	* @param pVmxTransient The VMX-transient structure.
4633	*
4634	* @remarks No-long-jump zone!!!
4635	*/
4636	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4637	{
4638	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4639	{
4640	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4641
4642	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4643	Let us assert it as such and use 32-bit VMWRITE. */
4644	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4645	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4646	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4647	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4648
4649	/*
4650	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4651	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4652	* can run the real-mode guest code under Virtual 8086 mode.
4653	*/
4654	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4655	if (pVmcsInfo->RealMode.fRealOnV86Active)
4656	{
4657	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4658	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4659	Assert(!pVmxTransient->fIsNestedGuest);
4660	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4661	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4662	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4663	}
4664
4665	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4666	AssertRCReturn(rc, rc);
4667
4668	/*
4669	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4670	*
4671	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4672	* through the hypervisor debugger using EFLAGS.TF.
4673	*/
4674	if ( !pVmxTransient->fIsNestedGuest
4675	&& !pVCpu->hm.s.fSingleInstruction
4676	&& fEFlags.Bits.u1TF)
4677	{
4678	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4679	* premature trips to ring-3 esp since IEM does not yet handle it. */
4680	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4681	AssertRCReturn(rc, rc);
4682	}
4683	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4684	* nested-guest VMCS. */
4685
4686	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4687	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4688	}
4689	return VINF_SUCCESS;
4690	}
4691
4692
4693	/**
4694	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4695	*
4696	* The guest FPU state is always pre-loaded hence we don't need to bother about
4697	* sharing FPU related CR0 bits between the guest and host.
4698	*
4699	* @returns VBox status code.
4700	* @param pVCpu The cross context virtual CPU structure.
4701	* @param pVmxTransient The VMX-transient structure.
4702	*
4703	* @remarks No-long-jump zone!!!
4704	*/
4705	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4706	{
4707	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4708	{
4709	PVM pVM = pVCpu->CTX_SUFF(pVM);
4710	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4711
4712	/*
4713	* Figure out fixed CR0 bits in VMX operation.
4714	*/
4715	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4716	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4717	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4718	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4719	else
4720	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4721
4722	if (!pVmxTransient->fIsNestedGuest)
4723	{
4724	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4725	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4726	uint64_t const u64ShadowCr0 = u64GuestCr0;
4727	Assert(!RT_HI_U32(u64GuestCr0));
4728
4729	/*
4730	* Setup VT-x's view of the guest CR0.
4731	*/
4732	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4733	if (pVM->hm.s.fNestedPaging)
4734	{
4735	if (CPUMIsGuestPagingEnabled(pVCpu))
4736	{
4737	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4738	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4739	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4740	}
4741	else
4742	{
4743	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4744	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4745	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4746	}
4747
4748	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4749	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4750	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4751	}
4752	else
4753	{
4754	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4755	u64GuestCr0 \|= X86_CR0_WP;
4756	}
4757
4758	/*
4759	* Guest FPU bits.
4760	*
4761	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4762	* using CR0.TS.
4763	*
4764	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4765	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4766	*/
4767	u64GuestCr0 \|= X86_CR0_NE;
4768
4769	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4770	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4771
4772	/*
4773	* Update exception intercepts.
4774	*/
4775	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4776	if (pVmcsInfo->RealMode.fRealOnV86Active)
4777	{
4778	Assert(PDMVmmDevHeapIsEnabled(pVM));
4779	Assert(pVM->hm.s.vmx.pRealModeTSS);
4780	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4781	}
4782	else
4783	{
4784	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4785	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4786	if (fInterceptMF)
4787	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4788	}
4789
4790	/* Additional intercepts for debugging, define these yourself explicitly. */
4791	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4792	uXcptBitmap \|= 0
4793	\| RT_BIT(X86_XCPT_BP)
4794	\| RT_BIT(X86_XCPT_DE)
4795	\| RT_BIT(X86_XCPT_NM)
4796	\| RT_BIT(X86_XCPT_TS)
4797	\| RT_BIT(X86_XCPT_UD)
4798	\| RT_BIT(X86_XCPT_NP)
4799	\| RT_BIT(X86_XCPT_SS)
4800	\| RT_BIT(X86_XCPT_GP)
4801	\| RT_BIT(X86_XCPT_PF)
4802	\| RT_BIT(X86_XCPT_MF)
4803	;
4804	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4805	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4806	#endif
4807	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4808	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4809	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4810
4811	/* Apply the fixed CR0 bits and enable caching. */
4812	u64GuestCr0 \|= fSetCr0;
4813	u64GuestCr0 &= fZapCr0;
4814	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4815
4816	/* Commit the CR0 and related fields to the guest VMCS. */
4817	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4818	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4819	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4820	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4821	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4822	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4823	AssertRCReturn(rc, rc);
4824
4825	/* Update our caches. */
4826	pVmcsInfo->u32ProcCtls = uProcCtls;
4827	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4828
4829	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4830	}
4831	else
4832	{
4833	/*
4834	* With nested-guests, we may have extended the guest/host mask here (since we
4835	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
4836	* mask can include more bits (to read from the nested-guest CR0 read-shadow) than
4837	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
4838	* those bits from the nested-guest CR0 into the nested-guest CR0 read shadow.
4839	*/
4840	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4841	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4842	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx);
4843	Assert(!RT_HI_U32(u64GuestCr0));
4844	Assert(u64GuestCr0 & X86_CR0_NE);
4845
4846	/* Apply the fixed CR0 bits and enable caching. */
4847	u64GuestCr0 \|= fSetCr0;
4848	u64GuestCr0 &= fZapCr0;
4849	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4850
4851	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4852	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4853	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4854	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4855	AssertRCReturn(rc, rc);
4856
4857	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4858	}
4859
4860	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4861	}
4862
4863	return VINF_SUCCESS;
4864	}
4865
4866
4867	/**
4868	* Exports the guest control registers (CR3, CR4) into the guest-state area
4869	* in the VMCS.
4870	*
4871	* @returns VBox strict status code.
4872	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4873	* without unrestricted guest access and the VMMDev is not presently
4874	* mapped (e.g. EFI32).
4875	*
4876	* @param pVCpu The cross context virtual CPU structure.
4877	* @param pVmxTransient The VMX-transient structure.
4878	*
4879	* @remarks No-long-jump zone!!!
4880	*/
4881	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4882	{
4883	int rc = VINF_SUCCESS;
4884	PVM pVM = pVCpu->CTX_SUFF(pVM);
4885
4886	/*
4887	* Guest CR2.
4888	* It's always loaded in the assembler code. Nothing to do here.
4889	*/
4890
4891	/*
4892	* Guest CR3.
4893	*/
4894	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4895	{
4896	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4897
4898	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4899	if (pVM->hm.s.fNestedPaging)
4900	{
4901	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4902	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4903
4904	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4905	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4906	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4907	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4908
4909	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4910	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4911	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4912
4913	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4914	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4915	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4916	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4917	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4918	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4919	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4920
4921	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4922	AssertRCReturn(rc, rc);
4923
4924	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4925	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4926	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4927	{
4928	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4929	if (CPUMIsGuestInPAEModeEx(pCtx))
4930	{
4931	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4932	AssertRCReturn(rc, rc);
4933	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4934	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4935	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4936	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4937	AssertRCReturn(rc, rc);
4938	}
4939
4940	/*
4941	* The guest's view of its CR3 is unblemished with nested paging when the
4942	* guest is using paging or we have unrestricted guest execution to handle
4943	* the guest when it's not using paging.
4944	*/
4945	GCPhysGuestCR3 = pCtx->cr3;
4946	}
4947	else
4948	{
4949	/*
4950	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4951	* thinks it accesses physical memory directly, we use our identity-mapped
4952	* page table to map guest-linear to guest-physical addresses. EPT takes care
4953	* of translating it to host-physical addresses.
4954	*/
4955	RTGCPHYS GCPhys;
4956	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4957
4958	/* We obtain it here every time as the guest could have relocated this PCI region. */
4959	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4960	if (RT_SUCCESS(rc))
4961	{ /* likely */ }
4962	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4963	{
4964	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4965	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4966	}
4967	else
4968	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4969
4970	GCPhysGuestCR3 = GCPhys;
4971	}
4972
4973	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4974	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4975	AssertRCReturn(rc, rc);
4976	}
4977	else
4978	{
4979	/* Non-nested paging case, just use the hypervisor's CR3. */
4980	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4981
4982	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4983	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4984	AssertRCReturn(rc, rc);
4985	}
4986
4987	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4988	}
4989
4990	/*
4991	* Guest CR4.
4992	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4993	*/
4994	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4995	{
4996	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4997	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4998
4999	/*
5000	* Figure out fixed CR4 bits in VMX operation.
5001	*/
5002	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5003	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5004
5005	/*
5006	* With nested-guests, we may have extended the guest/host mask here (since we
5007	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5008	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5009	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5010	* those bits from the nested-guest CR4 into the nested-guest CR4 read shadow.
5011	*/
5012	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5013	uint64_t u64GuestCr4 = pCtx->cr4;
5014	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5015	? pCtx->cr4
5016	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx);
5017	Assert(!RT_HI_U32(u64GuestCr4));
5018
5019	/*
5020	* Setup VT-x's view of the guest CR4.
5021	*
5022	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5023	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5024	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5025	*
5026	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5027	*/
5028	if (pVmcsInfo->RealMode.fRealOnV86Active)
5029	{
5030	Assert(pVM->hm.s.vmx.pRealModeTSS);
5031	Assert(PDMVmmDevHeapIsEnabled(pVM));
5032	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5033	}
5034
5035	if (pVM->hm.s.fNestedPaging)
5036	{
5037	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5038	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5039	{
5040	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5041	u64GuestCr4 \|= X86_CR4_PSE;
5042	/* Our identity mapping is a 32-bit page directory. */
5043	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5044	}
5045	/* else use guest CR4.*/
5046	}
5047	else
5048	{
5049	Assert(!pVmxTransient->fIsNestedGuest);
5050
5051	/*
5052	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5053	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5054	*/
5055	switch (pVCpu->hm.s.enmShadowMode)
5056	{
5057	case PGMMODE_REAL: /* Real-mode. */
5058	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5059	case PGMMODE_32_BIT: /* 32-bit paging. */
5060	{
5061	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5062	break;
5063	}
5064
5065	case PGMMODE_PAE: /* PAE paging. */
5066	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5067	{
5068	u64GuestCr4 \|= X86_CR4_PAE;
5069	break;
5070	}
5071
5072	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5073	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5074	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5075	break;
5076	#endif
5077	default:
5078	AssertFailed();
5079	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5080	}
5081	}
5082
5083	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5084	u64GuestCr4 \|= fSetCr4;
5085	u64GuestCr4 &= fZapCr4;
5086
5087	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5088	/** @todo Fix to 64-bit when we drop 32-bit. */
5089	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5090	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5091	AssertRCReturn(rc, rc);
5092
5093	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5094	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5095
5096	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5097
5098	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5099	}
5100	return rc;
5101	}
5102
5103
5104	/**
5105	* Exports the guest debug registers into the guest-state area in the VMCS.
5106	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5107	*
5108	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5109	*
5110	* @returns VBox status code.
5111	* @param pVCpu The cross context virtual CPU structure.
5112	* @param pVmxTransient The VMX-transient structure.
5113	*
5114	* @remarks No-long-jump zone!!!
5115	*/
5116	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5117	{
5118	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5119
5120	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5121	* stepping. */
5122	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5123	if (pVmxTransient->fIsNestedGuest)
5124	{
5125	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5126	AssertRCReturn(rc, rc);
5127	return VINF_SUCCESS;
5128	}
5129
5130	#ifdef VBOX_STRICT
5131	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5132	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5133	{
5134	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5135	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5136	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5137	}
5138	#endif
5139
5140	bool fSteppingDB = false;
5141	bool fInterceptMovDRx = false;
5142	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5143	if (pVCpu->hm.s.fSingleInstruction)
5144	{
5145	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5146	PVM pVM = pVCpu->CTX_SUFF(pVM);
5147	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5148	{
5149	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5150	Assert(fSteppingDB == false);
5151	}
5152	else
5153	{
5154	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5155	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5156	pVCpu->hm.s.fClearTrapFlag = true;
5157	fSteppingDB = true;
5158	}
5159	}
5160
5161	uint32_t u32GuestDr7;
5162	if ( fSteppingDB
5163	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5164	{
5165	/*
5166	* Use the combined guest and host DRx values found in the hypervisor register set
5167	* because the hypervisor debugger has breakpoints active or someone is single stepping
5168	* on the host side without a monitor trap flag.
5169	*
5170	* Note! DBGF expects a clean DR6 state before executing guest code.
5171	*/
5172	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5173	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5174	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5175	{
5176	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5177	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5178	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5179	}
5180	else
5181	#endif
5182	if (!CPUMIsHyperDebugStateActive(pVCpu))
5183	{
5184	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5185	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5186	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5187	}
5188
5189	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5190	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5191	pVCpu->hm.s.fUsingHyperDR7 = true;
5192	fInterceptMovDRx = true;
5193	}
5194	else
5195	{
5196	/*
5197	* If the guest has enabled debug registers, we need to load them prior to
5198	* executing guest code so they'll trigger at the right time.
5199	*/
5200	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5201	{
5202	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5203	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5204	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5205	{
5206	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5207	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5208	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5209	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5210	}
5211	else
5212	#endif
5213	if (!CPUMIsGuestDebugStateActive(pVCpu))
5214	{
5215	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5216	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5217	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5218	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5219	}
5220	Assert(!fInterceptMovDRx);
5221	}
5222	/*
5223	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5224	* must intercept #DB in order to maintain a correct DR6 guest value, and
5225	* because we need to intercept it to prevent nested #DBs from hanging the
5226	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5227	*/
5228	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5229	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5230	&& !CPUMIsGuestDebugStateActive(pVCpu))
5231	#else
5232	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5233	#endif
5234	{
5235	fInterceptMovDRx = true;
5236	}
5237
5238	/* Update DR7 with the actual guest value. */
5239	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5240	pVCpu->hm.s.fUsingHyperDR7 = false;
5241	}
5242
5243	if (fInterceptMovDRx)
5244	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5245	else
5246	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5247
5248	/*
5249	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5250	* monitor-trap flag and update our cache.
5251	*/
5252	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5253	{
5254	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5255	AssertRCReturn(rc2, rc2);
5256	pVmcsInfo->u32ProcCtls = uProcCtls;
5257	}
5258
5259	/*
5260	* Update guest DR7.
5261	*/
5262	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5263	AssertRCReturn(rc, rc);
5264
5265	/*
5266	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5267	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5268	*
5269	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5270	*/
5271	if (fSteppingDB)
5272	{
5273	Assert(pVCpu->hm.s.fSingleInstruction);
5274	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5275
5276	uint32_t fIntrState = 0;
5277	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5278	AssertRCReturn(rc, rc);
5279
5280	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5281	{
5282	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5283	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5284	AssertRCReturn(rc, rc);
5285	}
5286	}
5287
5288	return VINF_SUCCESS;
5289	}
5290
5291
5292	#ifdef VBOX_STRICT
5293	/**
5294	* Strict function to validate segment registers.
5295	*
5296	* @param pVCpu The cross context virtual CPU structure.
5297	* @param pVmcsInfo The VMCS info. object.
5298	*
5299	* @remarks Will import guest CR0 on strict builds during validation of
5300	* segments.
5301	*/
5302	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5303	{
5304	/*
5305	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5306	*
5307	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5308	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5309	* unusable bit and doesn't change the guest-context value.
5310	*/
5311	PVM pVM = pVCpu->CTX_SUFF(pVM);
5312	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5313	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5314	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5315	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5316	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5317	{
5318	/* Protected mode checks */
5319	/* CS */
5320	Assert(pCtx->cs.Attr.n.u1Present);
5321	Assert(!(pCtx->cs.Attr.u & 0xf00));
5322	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5323	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5324	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5325	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5326	\|\| (pCtx->cs.Attr.n.u1Granularity));
5327	/* CS cannot be loaded with NULL in protected mode. */
5328	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5329	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5330	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5331	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5332	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5333	else
5334	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5335	/* SS */
5336	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5337	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5338	if ( !(pCtx->cr0 & X86_CR0_PE)
5339	\|\| pCtx->cs.Attr.n.u4Type == 3)
5340	{
5341	Assert(!pCtx->ss.Attr.n.u2Dpl);
5342	}
5343	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5344	{
5345	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5346	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5347	Assert(pCtx->ss.Attr.n.u1Present);
5348	Assert(!(pCtx->ss.Attr.u & 0xf00));
5349	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5350	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5351	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5352	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5353	\|\| (pCtx->ss.Attr.n.u1Granularity));
5354	}
5355	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5356	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5357	{
5358	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5359	Assert(pCtx->ds.Attr.n.u1Present);
5360	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5361	Assert(!(pCtx->ds.Attr.u & 0xf00));
5362	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5363	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5364	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5365	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5366	\|\| (pCtx->ds.Attr.n.u1Granularity));
5367	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5368	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5369	}
5370	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5371	{
5372	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5373	Assert(pCtx->es.Attr.n.u1Present);
5374	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5375	Assert(!(pCtx->es.Attr.u & 0xf00));
5376	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5377	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5378	\|\| !(pCtx->es.Attr.n.u1Granularity));
5379	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5380	\|\| (pCtx->es.Attr.n.u1Granularity));
5381	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5382	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5383	}
5384	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5385	{
5386	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5387	Assert(pCtx->fs.Attr.n.u1Present);
5388	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5389	Assert(!(pCtx->fs.Attr.u & 0xf00));
5390	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5391	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5392	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5393	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5394	\|\| (pCtx->fs.Attr.n.u1Granularity));
5395	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5396	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5397	}
5398	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5399	{
5400	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5401	Assert(pCtx->gs.Attr.n.u1Present);
5402	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5403	Assert(!(pCtx->gs.Attr.u & 0xf00));
5404	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5405	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5406	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5407	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5408	\|\| (pCtx->gs.Attr.n.u1Granularity));
5409	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5410	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5411	}
5412	/* 64-bit capable CPUs. */
5413	# if HC_ARCH_BITS == 64
5414	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5415	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5416	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5417	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5418	# endif
5419	}
5420	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5421	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5422	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5423	{
5424	/* Real and v86 mode checks. */
5425	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5426	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5427	if (pVmcsInfo->RealMode.fRealOnV86Active)
5428	{
5429	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5430	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5431	}
5432	else
5433	{
5434	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5435	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5436	}
5437
5438	/* CS */
5439	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5440	Assert(pCtx->cs.u32Limit == 0xffff);
5441	Assert(u32CSAttr == 0xf3);
5442	/* SS */
5443	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5444	Assert(pCtx->ss.u32Limit == 0xffff);
5445	Assert(u32SSAttr == 0xf3);
5446	/* DS */
5447	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5448	Assert(pCtx->ds.u32Limit == 0xffff);
5449	Assert(u32DSAttr == 0xf3);
5450	/* ES */
5451	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5452	Assert(pCtx->es.u32Limit == 0xffff);
5453	Assert(u32ESAttr == 0xf3);
5454	/* FS */
5455	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5456	Assert(pCtx->fs.u32Limit == 0xffff);
5457	Assert(u32FSAttr == 0xf3);
5458	/* GS */
5459	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5460	Assert(pCtx->gs.u32Limit == 0xffff);
5461	Assert(u32GSAttr == 0xf3);
5462	/* 64-bit capable CPUs. */
5463	# if HC_ARCH_BITS == 64
5464	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5465	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5466	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5467	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5468	# endif
5469	}
5470	}
5471	#endif /* VBOX_STRICT */
5472
5473
5474	/**
5475	* Exports a guest segment register into the guest-state area in the VMCS.
5476	*
5477	* @returns VBox status code.
5478	* @param pVCpu The cross context virtual CPU structure.
5479	* @param pVmcsInfo The VMCS info. object.
5480	* @param iSegReg The segment register number (X86_SREG_XXX).
5481	* @param pSelReg Pointer to the segment selector.
5482	*
5483	* @remarks No-long-jump zone!!!
5484	*/
5485	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5486	{
5487	Assert(iSegReg < X86_SREG_COUNT);
5488	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5489	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5490	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5491	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5492
5493	uint32_t u32Access = pSelReg->Attr.u;
5494	if (pVmcsInfo->RealMode.fRealOnV86Active)
5495	{
5496	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5497	u32Access = 0xf3;
5498	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5499	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5500	RT_NOREF_PV(pVCpu);
5501	}
5502	else
5503	{
5504	/*
5505	* The way to differentiate between whether this is really a null selector or was just
5506	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5507	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5508	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5509	* NULL selectors loaded in protected-mode have their attribute as 0.
5510	*/
5511	if (!u32Access)
5512	u32Access = X86DESCATTR_UNUSABLE;
5513	}
5514
5515	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5516	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5517	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5518
5519	/*
5520	* Commit it to the VMCS.
5521	*/
5522	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5523	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5524	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5525	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5526	AssertRCReturn(rc, rc);
5527	return rc;
5528	}
5529
5530
5531	/**
5532	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5533	* area in the VMCS.
5534	*
5535	* @returns VBox status code.
5536	* @param pVCpu The cross context virtual CPU structure.
5537	* @param pVmxTransient The VMX-transient structure.
5538	*
5539	* @remarks Will import guest CR0 on strict builds during validation of
5540	* segments.
5541	* @remarks No-long-jump zone!!!
5542	*/
5543	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5544	{
5545	int rc = VERR_INTERNAL_ERROR_5;
5546	PVM pVM = pVCpu->CTX_SUFF(pVM);
5547	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5548	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5549
5550	/*
5551	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5552	*/
5553	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5554	{
5555	#ifdef VBOX_WITH_REM
5556	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5557	{
5558	Assert(!pVmxTransient->fIsNestedGuest);
5559	Assert(pVM->hm.s.vmx.pRealModeTSS);
5560	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5561	if ( pVmcsInfo->fWasInRealMode
5562	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5563	{
5564	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5565	in real-mode (e.g. OpenBSD 4.0) */
5566	REMFlushTBs(pVM);
5567	Log4Func(("Switch to protected mode detected!\n"));
5568	pVmcsInfo->fWasInRealMode = false;
5569	}
5570	}
5571	#endif
5572	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5573	{
5574	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5575	if (pVmcsInfo->RealMode.fRealOnV86Active)
5576	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5577	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5578	AssertRCReturn(rc, rc);
5579	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5580	}
5581
5582	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5583	{
5584	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5585	if (pVmcsInfo->RealMode.fRealOnV86Active)
5586	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5587	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5588	AssertRCReturn(rc, rc);
5589	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5590	}
5591
5592	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5593	{
5594	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5595	if (pVmcsInfo->RealMode.fRealOnV86Active)
5596	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5597	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5598	AssertRCReturn(rc, rc);
5599	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5600	}
5601
5602	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5603	{
5604	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5605	if (pVmcsInfo->RealMode.fRealOnV86Active)
5606	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5607	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5608	AssertRCReturn(rc, rc);
5609	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5610	}
5611
5612	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5613	{
5614	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5615	if (pVmcsInfo->RealMode.fRealOnV86Active)
5616	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5617	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5618	AssertRCReturn(rc, rc);
5619	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5620	}
5621
5622	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5623	{
5624	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5625	if (pVmcsInfo->RealMode.fRealOnV86Active)
5626	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5627	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5628	AssertRCReturn(rc, rc);
5629	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5630	}
5631
5632	#ifdef VBOX_STRICT
5633	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5634	#endif
5635	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5636	pCtx->cs.Attr.u));
5637	}
5638
5639	/*
5640	* Guest TR.
5641	*/
5642	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5643	{
5644	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5645
5646	/*
5647	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5648	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5649	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5650	*/
5651	uint16_t u16Sel;
5652	uint32_t u32Limit;
5653	uint64_t u64Base;
5654	uint32_t u32AccessRights;
5655	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5656	{
5657	u16Sel = pCtx->tr.Sel;
5658	u32Limit = pCtx->tr.u32Limit;
5659	u64Base = pCtx->tr.u64Base;
5660	u32AccessRights = pCtx->tr.Attr.u;
5661	}
5662	else
5663	{
5664	Assert(!pVmxTransient->fIsNestedGuest);
5665	Assert(pVM->hm.s.vmx.pRealModeTSS);
5666	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5667
5668	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5669	RTGCPHYS GCPhys;
5670	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5671	AssertRCReturn(rc, rc);
5672
5673	X86DESCATTR DescAttr;
5674	DescAttr.u = 0;
5675	DescAttr.n.u1Present = 1;
5676	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5677
5678	u16Sel = 0;
5679	u32Limit = HM_VTX_TSS_SIZE;
5680	u64Base = GCPhys;
5681	u32AccessRights = DescAttr.u;
5682	}
5683
5684	/* Validate. */
5685	Assert(!(u16Sel & RT_BIT(2)));
5686	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5687	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5688	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5689	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5690	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5691	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5692	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5693	Assert( (u32Limit & 0xfff) == 0xfff
5694	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5695	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5696	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5697
5698	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5699	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5700	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5701	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5702	AssertRCReturn(rc, rc);
5703
5704	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5705	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5706	}
5707
5708	/*
5709	* Guest GDTR.
5710	*/
5711	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5712	{
5713	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5714
5715	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5716	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5717	AssertRCReturn(rc, rc);
5718
5719	/* Validate. */
5720	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5721
5722	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5723	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5724	}
5725
5726	/*
5727	* Guest LDTR.
5728	*/
5729	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5730	{
5731	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5732
5733	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5734	uint32_t u32Access;
5735	if ( !pVmxTransient->fIsNestedGuest
5736	&& !pCtx->ldtr.Attr.u)
5737	u32Access = X86DESCATTR_UNUSABLE;
5738	else
5739	u32Access = pCtx->ldtr.Attr.u;
5740
5741	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5742	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5743	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5744	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5745	AssertRCReturn(rc, rc);
5746
5747	/* Validate. */
5748	if (!(u32Access & X86DESCATTR_UNUSABLE))
5749	{
5750	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5751	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5752	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5753	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5754	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5755	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5756	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5757	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5758	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5759	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5760	}
5761
5762	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5763	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5764	}
5765
5766	/*
5767	* Guest IDTR.
5768	*/
5769	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5770	{
5771	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5772
5773	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5774	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5775	AssertRCReturn(rc, rc);
5776
5777	/* Validate. */
5778	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5779
5780	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5781	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5782	}
5783
5784	return VINF_SUCCESS;
5785	}
5786
5787
5788	/**
5789	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5790	* areas.
5791	*
5792	* These MSRs will automatically be loaded to the host CPU on every successful
5793	* VM-entry and stored from the host CPU on every successful VM-exit.
5794	*
5795	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5796	* actual host MSR values are not- updated here for performance reasons. See
5797	* hmR0VmxExportHostMsrs().
5798	*
5799	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5800	*
5801	* @returns VBox status code.
5802	* @param pVCpu The cross context virtual CPU structure.
5803	* @param pVmxTransient The VMX-transient structure.
5804	*
5805	* @remarks No-long-jump zone!!!
5806	*/
5807	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5808	{
5809	AssertPtr(pVCpu);
5810	AssertPtr(pVmxTransient);
5811
5812	PVM pVM = pVCpu->CTX_SUFF(pVM);
5813	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5814
5815	/*
5816	* MSRs that we use the auto-load/store MSR area in the VMCS.
5817	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5818	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5819	*
5820	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5821	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5822	* emulation, nothing to do here.
5823	*/
5824	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5825	{
5826	if ( !pVmxTransient->fIsNestedGuest
5827	&& pVM->hm.s.fAllow64BitGuests)
5828	{
5829	#if HC_ARCH_BITS == 32
5830	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5831	Assert(!pVmxTransient->fIsNestedGuest);
5832
5833	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5834	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5835	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5836	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5837	AssertRCReturn(rc, rc);
5838	#endif
5839	}
5840	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5841	}
5842
5843	/*
5844	* Guest Sysenter MSRs.
5845	*/
5846	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5847	{
5848	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5849
5850	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5851	{
5852	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5853	AssertRCReturn(rc, rc);
5854	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5855	}
5856
5857	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5858	{
5859	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5860	AssertRCReturn(rc, rc);
5861	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5862	}
5863
5864	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5865	{
5866	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5867	AssertRCReturn(rc, rc);
5868	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5869	}
5870	}
5871
5872	/*
5873	* Guest/host EFER MSR.
5874	*/
5875	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5876	{
5877	/* Whether we are using the VMCS to swap the EFER MSR must have been
5878	determined earlier while exporting VM-entry/VM-exit controls. */
5879	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5880	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5881
5882	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5883	{
5884	/*
5885	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5886	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5887	*/
5888	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5889	{
5890	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5891	AssertRCReturn(rc, rc);
5892	}
5893	else
5894	{
5895	/*
5896	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5897	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5898	*/
5899	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5900	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5901	AssertRCReturn(rc, rc);
5902	}
5903	}
5904	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5905	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5906
5907	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5908	}
5909
5910	/*
5911	* Other MSRs.
5912	* Speculation Control (R/W).
5913	*/
5914	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5915	{
5916	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5917	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5918	{
5919	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5920	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5921	AssertRCReturn(rc, rc);
5922	}
5923	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5924	}
5925
5926	return VINF_SUCCESS;
5927	}
5928
5929
5930	/**
5931	* Selects up the appropriate function to run guest code.
5932	*
5933	* @returns VBox status code.
5934	* @param pVCpu The cross context virtual CPU structure.
5935	* @param pVmxTransient The VMX-transient structure.
5936	*
5937	* @remarks No-long-jump zone!!!
5938	*/
5939	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5940	{
5941	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5942	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5943
5944	if (CPUMIsGuestInLongModeEx(pCtx))
5945	{
5946	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5947	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5948	#endif
5949	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5950	#if HC_ARCH_BITS == 32
5951	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5952	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5953	{
5954	#ifdef VBOX_STRICT
5955	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5956	{
5957	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5958	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5959	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5960	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5961	("fCtxChanged=%#RX64\n", fCtxChanged));
5962	}
5963	#endif
5964	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5965
5966	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5967	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5968	pVmcsInfo->fSwitchedTo64on32 = true;
5969	Log4Func(("Selected 64-bit switcher\n"));
5970	}
5971	#else
5972	/* 64-bit host. */
5973	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5974	#endif
5975	}
5976	else
5977	{
5978	/* Guest is not in long mode, use the 32-bit handler. */
5979	#if HC_ARCH_BITS == 32
5980	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5981	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5982	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5983	{
5984	# ifdef VBOX_STRICT
5985	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5986	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5987	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5988	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5989	("fCtxChanged=%#RX64\n", fCtxChanged));
5990	# endif
5991	}
5992	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5993	/*
5994	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5995	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5996	* switcher flag now because we know the guest is in a sane state where it's safe
5997	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5998	* the much faster 32-bit switcher again.
5999	*/
6000	if (!pVmcsInfo->fSwitchedTo64on32)
6001	{
6002	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6003	Log4Func(("Selected 32-bit switcher\n"));
6004	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6005	}
6006	else
6007	{
6008	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6009	if ( pVmcsInfo->RealMode.fRealOnV86Active
6010	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6011	{
6012	pVmcsInfo->fSwitchedTo64on32 = false;
6013	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6014	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6015	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6016	\| HM_CHANGED_HOST_CONTEXT);
6017	Log4Func(("Selected 32-bit switcher (safe)\n"));
6018	}
6019	}
6020	# else
6021	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6022	# endif
6023	#else
6024	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6025	#endif
6026	}
6027	Assert(pVmcsInfo->pfnStartVM);
6028	return VINF_SUCCESS;
6029	}
6030
6031
6032	/**
6033	* Wrapper for running the guest code in VT-x.
6034	*
6035	* @returns VBox status code, no informational status codes.
6036	* @param pVCpu The cross context virtual CPU structure.
6037	* @param pVmxTransient The VMX-transient structure.
6038	*
6039	* @remarks No-long-jump zone!!!
6040	*/
6041	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6042	{
6043	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6044	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6045	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6046
6047	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6048
6049	/*
6050	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6051	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6052	* callee-saved and thus the need for this XMM wrapper.
6053	*
6054	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6055	*/
6056	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6057	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6058	PVM pVM = pVCpu->CTX_SUFF(pVM);
6059	#ifdef VBOX_WITH_KERNEL_USING_XMM
6060	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6061	#else
6062	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6063	#endif
6064	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6065	return rc;
6066	}
6067
6068
6069	/**
6070	* Reports world-switch error and dumps some useful debug info.
6071	*
6072	* @param pVCpu The cross context virtual CPU structure.
6073	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6074	* @param pVmxTransient The VMX-transient structure (only
6075	* exitReason updated).
6076	*/
6077	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6078	{
6079	Assert(pVCpu);
6080	Assert(pVmxTransient);
6081	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6082
6083	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6084	switch (rcVMRun)
6085	{
6086	case VERR_VMX_INVALID_VMXON_PTR:
6087	AssertFailed();
6088	break;
6089	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6090	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6091	{
6092	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6093	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6094	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6095	AssertRC(rc);
6096
6097	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6098	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6099	Cannot do it here as we may have been long preempted. */
6100
6101	#ifdef VBOX_STRICT
6102	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6103	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6104	pVmxTransient->uExitReason));
6105	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6106	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6107	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6108	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6109	else
6110	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6111	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6112	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6113
6114	/* VMX control bits. */
6115	uint32_t u32Val;
6116	uint64_t u64Val;
6117	RTHCUINTREG uHCReg;
6118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6119	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6121	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6122	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6123	{
6124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6125	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6126	}
6127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6128	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6129	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6130	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6131	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6132	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6133	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6134	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6135	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6136	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6137	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6138	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6139	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6140	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6141	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6142	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6143	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6144	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6145	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6146	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6147	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6148	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6149	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6150	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6151	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6152	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6153	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6154	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6155	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6156	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6157	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6158	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6159	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6160	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6161	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6162	{
6163	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6164	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6165	}
6166
6167	/* Guest bits. */
6168	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6169	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6170	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6171	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6172	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6173	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6174	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6175	{
6176	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6177	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6178	}
6179
6180	/* Host bits. */
6181	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6182	Log4(("Host CR0 %#RHr\n", uHCReg));
6183	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6184	Log4(("Host CR3 %#RHr\n", uHCReg));
6185	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6186	Log4(("Host CR4 %#RHr\n", uHCReg));
6187
6188	RTGDTR HostGdtr;
6189	PCX86DESCHC pDesc;
6190	ASMGetGDTR(&HostGdtr);
6191	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6192	Log4(("Host CS %#08x\n", u32Val));
6193	if (u32Val < HostGdtr.cbGdt)
6194	{
6195	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6196	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6197	}
6198
6199	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6200	Log4(("Host DS %#08x\n", u32Val));
6201	if (u32Val < HostGdtr.cbGdt)
6202	{
6203	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6204	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6205	}
6206
6207	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6208	Log4(("Host ES %#08x\n", u32Val));
6209	if (u32Val < HostGdtr.cbGdt)
6210	{
6211	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6212	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6213	}
6214
6215	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6216	Log4(("Host FS %#08x\n", u32Val));
6217	if (u32Val < HostGdtr.cbGdt)
6218	{
6219	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6220	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6221	}
6222
6223	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6224	Log4(("Host GS %#08x\n", u32Val));
6225	if (u32Val < HostGdtr.cbGdt)
6226	{
6227	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6228	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6229	}
6230
6231	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6232	Log4(("Host SS %#08x\n", u32Val));
6233	if (u32Val < HostGdtr.cbGdt)
6234	{
6235	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6236	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6237	}
6238
6239	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6240	Log4(("Host TR %#08x\n", u32Val));
6241	if (u32Val < HostGdtr.cbGdt)
6242	{
6243	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6244	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6245	}
6246
6247	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6248	Log4(("Host TR Base %#RHv\n", uHCReg));
6249	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6250	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6251	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6252	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6253	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6254	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6255	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6256	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6257	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6258	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6259	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6260	Log4(("Host RSP %#RHv\n", uHCReg));
6261	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6262	Log4(("Host RIP %#RHv\n", uHCReg));
6263	# if HC_ARCH_BITS == 64
6264	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6265	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6266	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6267	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6268	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6269	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6270	# endif
6271	#endif /* VBOX_STRICT */
6272	break;
6273	}
6274
6275	default:
6276	/* Impossible */
6277	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6278	break;
6279	}
6280	}
6281
6282
6283	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6284	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6285	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6286	# endif
6287
6288	/**
6289	* Initialize the VMCS-Read cache.
6290	*
6291	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6292	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6293	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6294	* (those that have a 32-bit FULL & HIGH part).
6295	*
6296	* @param pVCpu The cross context virtual CPU structure.
6297	*/
6298	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6299	{
6300	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6301	do { \
6302	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6303	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6304	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6305	++cReadFields; \
6306	} while (0)
6307
6308	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6309	uint32_t cReadFields = 0;
6310
6311	/*
6312	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6313	* and serve to indicate exceptions to the rules.
6314	*/
6315
6316	/* Guest-natural selector base fields. */
6317	#if 0
6318	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6321	#endif
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6334	#if 0
6335	/* Unused natural width guest-state fields. */
6336	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6337	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6338	#endif
6339	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6340	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6341
6342	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6343	these 64-bit fields (using "FULL" and "HIGH" fields). */
6344	#if 0
6345	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6346	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6347	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6348	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6349	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6350	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6351	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6352	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6353	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6354	#endif
6355
6356	/* Natural width guest-state fields. */
6357	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6358	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6359
6360	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6361	{
6362	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6363	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6364	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6365	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6366	}
6367	else
6368	{
6369	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6370	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6371	}
6372
6373	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6374	}
6375
6376
6377	/**
6378	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6379	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6380	* darwin, running 64-bit guests).
6381	*
6382	* @returns VBox status code.
6383	* @param pVCpu The cross context virtual CPU structure.
6384	* @param idxField The VMCS field encoding.
6385	* @param u64Val 16, 32 or 64-bit value.
6386	*/
6387	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6388	{
6389	int rc;
6390	switch (idxField)
6391	{
6392	/*
6393	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6394	*/
6395	/* 64-bit Control fields. */
6396	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6397	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6398	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6399	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6400	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6401	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6402	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6403	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6404	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6405	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6406	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6407	case VMX_VMCS64_CTRL_EPTP_FULL:
6408	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6409	/* 64-bit Guest-state fields. */
6410	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6411	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6412	case VMX_VMCS64_GUEST_PAT_FULL:
6413	case VMX_VMCS64_GUEST_EFER_FULL:
6414	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6415	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6416	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6417	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6418	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6419	/* 64-bit Host-state fields. */
6420	case VMX_VMCS64_HOST_PAT_FULL:
6421	case VMX_VMCS64_HOST_EFER_FULL:
6422	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6423	{
6424	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6425	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6426	break;
6427	}
6428
6429	/*
6430	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6431	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6432	*/
6433	/* Natural-width Guest-state fields. */
6434	case VMX_VMCS_GUEST_CR3:
6435	case VMX_VMCS_GUEST_ES_BASE:
6436	case VMX_VMCS_GUEST_CS_BASE:
6437	case VMX_VMCS_GUEST_SS_BASE:
6438	case VMX_VMCS_GUEST_DS_BASE:
6439	case VMX_VMCS_GUEST_FS_BASE:
6440	case VMX_VMCS_GUEST_GS_BASE:
6441	case VMX_VMCS_GUEST_LDTR_BASE:
6442	case VMX_VMCS_GUEST_TR_BASE:
6443	case VMX_VMCS_GUEST_GDTR_BASE:
6444	case VMX_VMCS_GUEST_IDTR_BASE:
6445	case VMX_VMCS_GUEST_RSP:
6446	case VMX_VMCS_GUEST_RIP:
6447	case VMX_VMCS_GUEST_SYSENTER_ESP:
6448	case VMX_VMCS_GUEST_SYSENTER_EIP:
6449	{
6450	if (!(RT_HI_U32(u64Val)))
6451	{
6452	/* If this field is 64-bit, VT-x will zero out the top bits. */
6453	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6454	}
6455	else
6456	{
6457	/* Assert that only the 32->64 switcher case should ever come here. */
6458	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6459	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6460	}
6461	break;
6462	}
6463
6464	default:
6465	{
6466	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6467	pVCpu->hm.s.u32HMError = idxField;
6468	rc = VERR_INVALID_PARAMETER;
6469	break;
6470	}
6471	}
6472	AssertRCReturn(rc, rc);
6473	return rc;
6474	}
6475
6476
6477	/**
6478	* Queue up a VMWRITE by using the VMCS write cache.
6479	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6480	*
6481	* @param pVCpu The cross context virtual CPU structure.
6482	* @param idxField The VMCS field encoding.
6483	* @param u64Val 16, 32 or 64-bit value.
6484	*/
6485	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6486	{
6487	AssertPtr(pVCpu);
6488	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6489
6490	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6491	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6492
6493	/* Make sure there are no duplicates. */
6494	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6495	{
6496	if (pCache->Write.aField[i] == idxField)
6497	{
6498	pCache->Write.aFieldVal[i] = u64Val;
6499	return VINF_SUCCESS;
6500	}
6501	}
6502
6503	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6504	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6505	pCache->Write.cValidEntries++;
6506	return VINF_SUCCESS;
6507	}
6508	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6509
6510
6511	/**
6512	* Sets up the usage of TSC-offsetting and updates the VMCS.
6513	*
6514	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6515	* VMX-preemption timer.
6516	*
6517	* @returns VBox status code.
6518	* @param pVCpu The cross context virtual CPU structure.
6519	* @param pVmxTransient The VMX-transient structure.
6520	*
6521	* @remarks No-long-jump zone!!!
6522	*/
6523	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6524	{
6525	bool fOffsettedTsc;
6526	bool fParavirtTsc;
6527	uint64_t uTscOffset;
6528	PVM pVM = pVCpu->CTX_SUFF(pVM);
6529	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6530
6531	if (pVM->hm.s.vmx.fUsePreemptTimer)
6532	{
6533	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6534
6535	/* Make sure the returned values have sane upper and lower boundaries. */
6536	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6537	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6538	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6539	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6540
6541	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6542	* preemption timers here. We probably need to clamp the preemption timer,
6543	* after converting the timer value to the host. */
6544	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6545	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6546	AssertRC(rc);
6547	}
6548	else
6549	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6550
6551	if (fParavirtTsc)
6552	{
6553	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6554	information before every VM-entry, hence disable it for performance sake. */
6555	#if 0
6556	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6557	AssertRC(rc);
6558	#endif
6559	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6560	}
6561
6562	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6563	if ( fOffsettedTsc
6564	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6565	{
6566	if (pVmxTransient->fIsNestedGuest)
6567	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6568	if (pVmcsInfo->u64TscOffset != uTscOffset)
6569	{
6570	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6571	AssertRC(rc);
6572	pVmcsInfo->u64TscOffset = uTscOffset;
6573	}
6574
6575	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6576	{
6577	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6578	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6579	AssertRC(rc);
6580	pVmcsInfo->u32ProcCtls = uProcCtls;
6581	}
6582	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6583	}
6584	else
6585	{
6586	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6587	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6588	{
6589	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6590	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6591	AssertRC(rc);
6592	pVmcsInfo->u32ProcCtls = uProcCtls;
6593	}
6594	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6595	}
6596	}
6597
6598
6599	/**
6600	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6601	* VM-exit interruption info type.
6602	*
6603	* @returns The IEM exception flags.
6604	* @param uVector The event vector.
6605	* @param uVmxEventType The VMX event type.
6606	*
6607	* @remarks This function currently only constructs flags required for
6608	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6609	* and CR2 aspects of an exception are not included).
6610	*/
6611	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6612	{
6613	uint32_t fIemXcptFlags;
6614	switch (uVmxEventType)
6615	{
6616	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6617	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6618	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6619	break;
6620
6621	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6622	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6623	break;
6624
6625	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6626	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6627	break;
6628
6629	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6630	{
6631	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6632	if (uVector == X86_XCPT_BP)
6633	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6634	else if (uVector == X86_XCPT_OF)
6635	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6636	else
6637	{
6638	fIemXcptFlags = 0;
6639	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6640	}
6641	break;
6642	}
6643
6644	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6645	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6646	break;
6647
6648	default:
6649	fIemXcptFlags = 0;
6650	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6651	break;
6652	}
6653	return fIemXcptFlags;
6654	}
6655
6656
6657	/**
6658	* Sets an event as a pending event to be injected into the guest.
6659	*
6660	* @param pVCpu The cross context virtual CPU structure.
6661	* @param u32IntInfo The VM-entry interruption-information field.
6662	* @param cbInstr The VM-entry instruction length in bytes (for software
6663	* interrupts, exceptions and privileged software
6664	* exceptions).
6665	* @param u32ErrCode The VM-entry exception error code.
6666	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6667	* page-fault.
6668	*/
6669	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6670	RTGCUINTPTR GCPtrFaultAddress)
6671	{
6672	Assert(!pVCpu->hm.s.Event.fPending);
6673	pVCpu->hm.s.Event.fPending = true;
6674	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6675	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6676	pVCpu->hm.s.Event.cbInstr = cbInstr;
6677	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6678	}
6679
6680
6681	/**
6682	* Sets an external interrupt as pending-for-injection into the VM.
6683	*
6684	* @param pVCpu The cross context virtual CPU structure.
6685	* @param u8Interrupt The external interrupt vector.
6686	*/
6687	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6688	{
6689	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6690	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6691	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6692	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6693	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6694	}
6695
6696
6697	/**
6698	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6699	*
6700	* @param pVCpu The cross context virtual CPU structure.
6701	*/
6702	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6703	{
6704	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6706	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6707	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6708	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6709	}
6710
6711
6712	/**
6713	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6714	*
6715	* @param pVCpu The cross context virtual CPU structure.
6716	*/
6717	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6718	{
6719	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6720	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6721	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6722	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6723	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6724	}
6725
6726
6727	/**
6728	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6729	*
6730	* @param pVCpu The cross context virtual CPU structure.
6731	*/
6732	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6733	{
6734	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6735	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6736	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6737	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6738	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6739	}
6740
6741
6742	/**
6743	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6744	*
6745	* @param pVCpu The cross context virtual CPU structure.
6746	*/
6747	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6748	{
6749	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6751	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6752	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6753	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6754	}
6755
6756
6757	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6758	/**
6759	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6760	*
6761	* @param pVCpu The cross context virtual CPU structure.
6762	* @param u32ErrCode The error code for the general-protection exception.
6763	*/
6764	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6765	{
6766	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6767	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6768	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6769	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6770	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6771	}
6772
6773
6774	/**
6775	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6776	*
6777	* @param pVCpu The cross context virtual CPU structure.
6778	* @param u32ErrCode The error code for the stack exception.
6779	*/
6780	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6781	{
6782	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6783	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6784	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6785	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6786	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6787	}
6788
6789
6790	/**
6791	* Decodes the memory operand of an instruction that caused a VM-exit.
6792	*
6793	* The VM-exit qualification field provides the displacement field for memory
6794	* operand instructions, if any.
6795	*
6796	* @returns Strict VBox status code (i.e. informational status codes too).
6797	* @retval VINF_SUCCESS if the operand was successfully decoded.
6798	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6799	* operand.
6800	* @param pVCpu The cross context virtual CPU structure.
6801	* @param uExitInstrInfo The VM-exit instruction information field.
6802	* @param enmMemAccess The memory operand's access type (read or write).
6803	* @param GCPtrDisp The instruction displacement field, if any. For
6804	* RIP-relative addressing pass RIP + displacement here.
6805	* @param pGCPtrMem Where to store the effective destination memory address.
6806	*
6807	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6808	* virtual-8086 mode hence skips those checks while verifying if the
6809	* segment is valid.
6810	*/
6811	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6812	PRTGCPTR pGCPtrMem)
6813	{
6814	Assert(pGCPtrMem);
6815	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6816	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6817	\| CPUMCTX_EXTRN_CR0);
6818
6819	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6820	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6821	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6822
6823	VMXEXITINSTRINFO ExitInstrInfo;
6824	ExitInstrInfo.u = uExitInstrInfo;
6825	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6826	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6827	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6828	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6829	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6830	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6831	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6832	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6833	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6834
6835	/*
6836	* Validate instruction information.
6837	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6838	*/
6839	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6840	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6841	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6842	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6843	AssertLogRelMsgReturn(fIsMemOperand,
6844	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6845
6846	/*
6847	* Compute the complete effective address.
6848	*
6849	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6850	* See AMD spec. 4.5.2 "Segment Registers".
6851	*/
6852	RTGCPTR GCPtrMem = GCPtrDisp;
6853	if (fBaseRegValid)
6854	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6855	if (fIdxRegValid)
6856	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6857
6858	RTGCPTR const GCPtrOff = GCPtrMem;
6859	if ( !fIsLongMode
6860	\|\| iSegReg >= X86_SREG_FS)
6861	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6862	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6863
6864	/*
6865	* Validate effective address.
6866	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6867	*/
6868	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6869	Assert(cbAccess > 0);
6870	if (fIsLongMode)
6871	{
6872	if (X86_IS_CANONICAL(GCPtrMem))
6873	{
6874	*pGCPtrMem = GCPtrMem;
6875	return VINF_SUCCESS;
6876	}
6877
6878	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6879	* "Data Limit Checks in 64-bit Mode". */
6880	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6881	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6882	return VINF_HM_PENDING_XCPT;
6883	}
6884
6885	/*
6886	* This is a watered down version of iemMemApplySegment().
6887	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6888	* and segment CPL/DPL checks are skipped.
6889	*/
6890	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6891	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6892	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6893
6894	/* Check if the segment is present and usable. */
6895	if ( pSel->Attr.n.u1Present
6896	&& !pSel->Attr.n.u1Unusable)
6897	{
6898	Assert(pSel->Attr.n.u1DescType);
6899	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6900	{
6901	/* Check permissions for the data segment. */
6902	if ( enmMemAccess == VMXMEMACCESS_WRITE
6903	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6904	{
6905	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6906	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6907	return VINF_HM_PENDING_XCPT;
6908	}
6909
6910	/* Check limits if it's a normal data segment. */
6911	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6912	{
6913	if ( GCPtrFirst32 > pSel->u32Limit
6914	\|\| GCPtrLast32 > pSel->u32Limit)
6915	{
6916	Log4Func(("Data segment limit exceeded. "
6917	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6918	GCPtrLast32, pSel->u32Limit));
6919	if (iSegReg == X86_SREG_SS)
6920	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6921	else
6922	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6923	return VINF_HM_PENDING_XCPT;
6924	}
6925	}
6926	else
6927	{
6928	/* Check limits if it's an expand-down data segment.
6929	Note! The upper boundary is defined by the B bit, not the G bit! */
6930	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6931	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6932	{
6933	Log4Func(("Expand-down data segment limit exceeded. "
6934	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6935	GCPtrLast32, pSel->u32Limit));
6936	if (iSegReg == X86_SREG_SS)
6937	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6938	else
6939	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6940	return VINF_HM_PENDING_XCPT;
6941	}
6942	}
6943	}
6944	else
6945	{
6946	/* Check permissions for the code segment. */
6947	if ( enmMemAccess == VMXMEMACCESS_WRITE
6948	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6949	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6950	{
6951	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6952	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6953	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6954	return VINF_HM_PENDING_XCPT;
6955	}
6956
6957	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6958	if ( GCPtrFirst32 > pSel->u32Limit
6959	\|\| GCPtrLast32 > pSel->u32Limit)
6960	{
6961	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6962	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6963	if (iSegReg == X86_SREG_SS)
6964	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6965	else
6966	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6967	return VINF_HM_PENDING_XCPT;
6968	}
6969	}
6970	}
6971	else
6972	{
6973	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6974	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6975	return VINF_HM_PENDING_XCPT;
6976	}
6977
6978	*pGCPtrMem = GCPtrMem;
6979	return VINF_SUCCESS;
6980	}
6981
6982
6983	/**
6984	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6985	* guest attempting to execute a VMX instruction.
6986	*
6987	* @returns Strict VBox status code (i.e. informational status codes too).
6988	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6989	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6990	*
6991	* @param pVCpu The cross context virtual CPU structure.
6992	* @param uExitReason The VM-exit reason.
6993	*
6994	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6995	* @remarks No-long-jump zone!!!
6996	*/
6997	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6998	{
6999	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
7000	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7001
7002	/*
7003	* The physical CPU would have already checked the CPU mode/code segment.
7004	* We shall just assert here for paranoia.
7005	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7006	*/
7007	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7008	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7009	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7010
7011	if (uExitReason == VMX_EXIT_VMXON)
7012	{
7013	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7014
7015	/*
7016	* We check CR4.VMXE because it is required to be always set while in VMX operation
7017	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
7018	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7019	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7020	*/
7021	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7022	{
7023	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7024	hmR0VmxSetPendingXcptUD(pVCpu);
7025	return VINF_HM_PENDING_XCPT;
7026	}
7027	}
7028	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7029	{
7030	/*
7031	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7032	* (other than VMXON), we need to raise a #UD.
7033	*/
7034	Log4Func(("Not in VMX root mode -> #UD\n"));
7035	hmR0VmxSetPendingXcptUD(pVCpu);
7036	return VINF_HM_PENDING_XCPT;
7037	}
7038
7039	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7040	return VINF_SUCCESS;
7041	}
7042	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7043
7044
7045	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7046	{
7047	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7048
7049	/*
7050	* If VT-x marks the segment as unusable, most other bits remain undefined:
7051	* - For CS the L, D and G bits have meaning.
7052	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7053	* - For the remaining data segments no bits are defined.
7054	*
7055	* The present bit and the unusable bit has been observed to be set at the
7056	* same time (the selector was supposed to be invalid as we started executing
7057	* a V8086 interrupt in ring-0).
7058	*
7059	* What should be important for the rest of the VBox code, is that the P bit is
7060	* cleared. Some of the other VBox code recognizes the unusable bit, but
7061	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7062	* safe side here, we'll strip off P and other bits we don't care about. If
7063	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7064	*
7065	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7066	*/
7067	#ifdef VBOX_STRICT
7068	uint32_t const uAttr = pSelReg->Attr.u;
7069	#endif
7070
7071	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7072	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7073	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7074
7075	#ifdef VBOX_STRICT
7076	VMMRZCallRing3Disable(pVCpu);
7077	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7078	# ifdef DEBUG_bird
7079	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7080	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7081	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7082	# endif
7083	VMMRZCallRing3Enable(pVCpu);
7084	NOREF(uAttr);
7085	#endif
7086	RT_NOREF2(pVCpu, idxSel);
7087	}
7088
7089
7090	/**
7091	* Imports a guest segment register from the current VMCS into the guest-CPU
7092	* context.
7093	*
7094	* @returns VBox status code.
7095	* @param pVCpu The cross context virtual CPU structure.
7096	* @param iSegReg The segment register number (X86_SREG_XXX).
7097	*
7098	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7099	* do not log!
7100	*/
7101	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7102	{
7103	Assert(iSegReg < X86_SREG_COUNT);
7104
7105	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7106	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7107	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7108	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7109	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7110	#else
7111	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7112	#endif
7113	uint64_t u64Base;
7114	uint32_t u32Sel, u32Limit, u32Attr;
7115	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7116	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7117	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7118	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7119	if (RT_SUCCESS(rc))
7120	{
7121	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7122	pSelReg->Sel = u32Sel;
7123	pSelReg->ValidSel = u32Sel;
7124	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7125	pSelReg->u32Limit = u32Limit;
7126	pSelReg->u64Base = u64Base;
7127	pSelReg->Attr.u = u32Attr;
7128	if (u32Attr & X86DESCATTR_UNUSABLE)
7129	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7130	}
7131	return rc;
7132	}
7133
7134
7135	/**
7136	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7137	*
7138	* @returns VBox status code.
7139	* @param pVCpu The cross context virtual CPU structure.
7140	*
7141	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7142	* do not log!
7143	*/
7144	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7145	{
7146	uint64_t u64Base;
7147	uint32_t u32Sel, u32Limit, u32Attr;
7148	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7149	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7150	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7151	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7152	if (RT_SUCCESS(rc))
7153	{
7154	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7155	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7156	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7157	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7158	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7159	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7160	if (u32Attr & X86DESCATTR_UNUSABLE)
7161	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7162	}
7163	return rc;
7164	}
7165
7166
7167	/**
7168	* Imports the guest TR from the current VMCS into the guest-CPU context.
7169	*
7170	* @returns VBox status code.
7171	* @param pVCpu The cross context virtual CPU structure.
7172	*
7173	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7174	* do not log!
7175	*/
7176	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7177	{
7178	uint32_t u32Sel, u32Limit, u32Attr;
7179	uint64_t u64Base;
7180	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7181	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7182	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7183	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7184	AssertRCReturn(rc, rc);
7185
7186	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7187	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7188	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7189	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7190	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7191	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7192	/* TR is the only selector that can never be unusable. */
7193	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7194	return VINF_SUCCESS;
7195	}
7196
7197
7198	/**
7199	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7200	*
7201	* @returns VBox status code.
7202	* @param pVCpu The cross context virtual CPU structure.
7203	*
7204	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7205	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7206	* instead!!!
7207	*/
7208	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7209	{
7210	uint64_t u64Val;
7211	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7212	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7213	{
7214	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7215	if (RT_SUCCESS(rc))
7216	{
7217	pCtx->rip = u64Val;
7218	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7219	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7220	}
7221	return rc;
7222	}
7223	return VINF_SUCCESS;
7224	}
7225
7226
7227	/**
7228	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7229	*
7230	* @returns VBox status code.
7231	* @param pVCpu The cross context virtual CPU structure.
7232	* @param pVmcsInfo The VMCS info. object.
7233	*
7234	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7235	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7236	* instead!!!
7237	*/
7238	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7239	{
7240	uint32_t u32Val;
7241	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7242	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7243	{
7244	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7245	if (RT_SUCCESS(rc))
7246	{
7247	pCtx->eflags.u32 = u32Val;
7248
7249	/* Restore eflags for real-on-v86-mode hack. */
7250	if (pVmcsInfo->RealMode.fRealOnV86Active)
7251	{
7252	pCtx->eflags.Bits.u1VM = 0;
7253	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7254	}
7255	}
7256	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7257	return rc;
7258	}
7259	return VINF_SUCCESS;
7260	}
7261
7262
7263	/**
7264	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7265	* context.
7266	*
7267	* @returns VBox status code.
7268	* @param pVCpu The cross context virtual CPU structure.
7269	* @param pVmcsInfo The VMCS info. object.
7270	*
7271	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7272	* do not log!
7273	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7274	* instead!!!
7275	*/
7276	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7277	{
7278	uint32_t u32Val;
7279	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7280	if (RT_SUCCESS(rc))
7281	{
7282	if (!u32Val)
7283	{
7284	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7285	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7286
7287	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7288	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7289	}
7290	else
7291	{
7292	/*
7293	* We must import RIP here to set our EM interrupt-inhibited state.
7294	* We also import RFLAGS as our code that evaluates pending interrupts
7295	* before VM-entry requires it.
7296	*/
7297	rc = hmR0VmxImportGuestRip(pVCpu);
7298	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7299	if (RT_SUCCESS(rc))
7300	{
7301	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7302	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7303	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7304	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7305
7306	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7307	{
7308	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7309	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7310	}
7311	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7312	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7313	}
7314	}
7315	}
7316	return rc;
7317	}
7318
7319
7320	/**
7321	* Worker for VMXR0ImportStateOnDemand.
7322	*
7323	* @returns VBox status code.
7324	* @param pVCpu The cross context virtual CPU structure.
7325	* @param pVmcsInfo The VMCS info. object.
7326	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7327	*/
7328	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7329	{
7330	#define VMXLOCAL_BREAK_RC(a_rc) \
7331	if (RT_SUCCESS(a_rc)) \
7332	{ } \
7333	else \
7334	break
7335
7336	int rc = VINF_SUCCESS;
7337	PVM pVM = pVCpu->CTX_SUFF(pVM);
7338	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7339	uint64_t u64Val;
7340	uint32_t u32Val;
7341
7342	/*
7343	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7344	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7345	* neither are other host platforms.
7346	*
7347	* Committing this temporarily as it prevents BSOD.
7348	*
7349	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7350	*/
7351	#ifdef RT_OS_WINDOWS
7352	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7353	return VERR_HM_IPE_1;
7354	#endif
7355
7356	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7357
7358	/*
7359	* We disable interrupts to make the updating of the state and in particular
7360	* the fExtrn modification atomic wrt to preemption hooks.
7361	*/
7362	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7363
7364	fWhat &= pCtx->fExtrn;
7365	if (fWhat)
7366	{
7367	do
7368	{
7369	if (fWhat & CPUMCTX_EXTRN_RIP)
7370	{
7371	rc = hmR0VmxImportGuestRip(pVCpu);
7372	VMXLOCAL_BREAK_RC(rc);
7373	}
7374
7375	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7376	{
7377	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7378	VMXLOCAL_BREAK_RC(rc);
7379	}
7380
7381	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7382	{
7383	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7384	VMXLOCAL_BREAK_RC(rc);
7385	}
7386
7387	if (fWhat & CPUMCTX_EXTRN_RSP)
7388	{
7389	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7390	VMXLOCAL_BREAK_RC(rc);
7391	pCtx->rsp = u64Val;
7392	}
7393
7394	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7395	{
7396	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7397	if (fWhat & CPUMCTX_EXTRN_CS)
7398	{
7399	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7400	rc \|= hmR0VmxImportGuestRip(pVCpu);
7401	if (fRealOnV86Active)
7402	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7403	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7404	}
7405	if (fWhat & CPUMCTX_EXTRN_SS)
7406	{
7407	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7408	if (fRealOnV86Active)
7409	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7410	}
7411	if (fWhat & CPUMCTX_EXTRN_DS)
7412	{
7413	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7414	if (fRealOnV86Active)
7415	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7416	}
7417	if (fWhat & CPUMCTX_EXTRN_ES)
7418	{
7419	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7420	if (fRealOnV86Active)
7421	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7422	}
7423	if (fWhat & CPUMCTX_EXTRN_FS)
7424	{
7425	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7426	if (fRealOnV86Active)
7427	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7428	}
7429	if (fWhat & CPUMCTX_EXTRN_GS)
7430	{
7431	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7432	if (fRealOnV86Active)
7433	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7434	}
7435	VMXLOCAL_BREAK_RC(rc);
7436	}
7437
7438	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7439	{
7440	if (fWhat & CPUMCTX_EXTRN_LDTR)
7441	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7442
7443	if (fWhat & CPUMCTX_EXTRN_GDTR)
7444	{
7445	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7446	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7447	pCtx->gdtr.pGdt = u64Val;
7448	pCtx->gdtr.cbGdt = u32Val;
7449	}
7450
7451	/* Guest IDTR. */
7452	if (fWhat & CPUMCTX_EXTRN_IDTR)
7453	{
7454	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7455	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7456	pCtx->idtr.pIdt = u64Val;
7457	pCtx->idtr.cbIdt = u32Val;
7458	}
7459
7460	/* Guest TR. */
7461	if (fWhat & CPUMCTX_EXTRN_TR)
7462	{
7463	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7464	don't need to import that one. */
7465	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7466	rc \|= hmR0VmxImportGuestTr(pVCpu);
7467	}
7468	VMXLOCAL_BREAK_RC(rc);
7469	}
7470
7471	if (fWhat & CPUMCTX_EXTRN_DR7)
7472	{
7473	if (!pVCpu->hm.s.fUsingHyperDR7)
7474	{
7475	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7476	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7477	VMXLOCAL_BREAK_RC(rc);
7478	pCtx->dr[7] = u32Val;
7479	}
7480	}
7481
7482	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7483	{
7484	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7485	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7486	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7487	pCtx->SysEnter.cs = u32Val;
7488	VMXLOCAL_BREAK_RC(rc);
7489	}
7490
7491	#if HC_ARCH_BITS == 64
7492	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7493	{
7494	if ( pVM->hm.s.fAllow64BitGuests
7495	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7496	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7497	}
7498
7499	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7500	{
7501	if ( pVM->hm.s.fAllow64BitGuests
7502	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7503	{
7504	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7505	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7506	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7507	}
7508	}
7509	#endif
7510
7511	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7512	#if HC_ARCH_BITS == 32
7513	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7514	#endif
7515	)
7516	{
7517	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7518	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7519	Assert(pMsrs);
7520	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7521	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7522	for (uint32_t i = 0; i < cMsrs; i++)
7523	{
7524	uint32_t const idMsr = pMsrs[i].u32Msr;
7525	switch (idMsr)
7526	{
7527	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7528	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7529	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7530	#if HC_ARCH_BITS == 32
7531	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7532	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7533	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7534	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7535	#endif
7536	default:
7537	{
7538	pCtx->fExtrn = 0;
7539	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7540	ASMSetFlags(fEFlags);
7541	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7542	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7543	}
7544	}
7545	}
7546	}
7547
7548	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7549	{
7550	uint64_t u64Shadow;
7551	if (fWhat & CPUMCTX_EXTRN_CR0)
7552	{
7553	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7554	* remove when we drop 32-bit host w/ 64-bit host support, see
7555	* @bugref{9180#c39}. */
7556	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7557	#if HC_ARCH_BITS == 32
7558	uint32_t u32Shadow;
7559	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7560	u64Shadow = u32Shadow;
7561	#else
7562	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7563	#endif
7564	VMXLOCAL_BREAK_RC(rc);
7565	u64Val = u32Val;
7566	#if 1
7567	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7568	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7569	#else
7570	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7571	{
7572	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7573	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7574	}
7575	else
7576	{
7577	/** @todo NSTVMX: We need to do some unfudging here because we altered the
7578	* guest/host mask before running the nested-guest. */
7579	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7580	Assert(pVmcsNstGst);
7581
7582	uint64_t const uGstCr0Mask = pVmcsNstGst->u64Cr0Mask.u;
7583	uint64_t const uHstCr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
7584	}
7585	#endif
7586	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7587	CPUMSetGuestCR0(pVCpu, u64Val);
7588	VMMRZCallRing3Enable(pVCpu);
7589	}
7590
7591	if (fWhat & CPUMCTX_EXTRN_CR4)
7592	{
7593	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7594	* remove when we drop 32-bit host w/ 64-bit host support, see
7595	* @bugref{9180#c39}. */
7596	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7597	#if HC_ARCH_BITS == 32
7598	uint32_t u32Shadow;
7599	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7600	u64Shadow = u32Shadow;
7601	#else
7602	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7603	#endif
7604	VMXLOCAL_BREAK_RC(rc);
7605	u64Val = u32Val;
7606	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7607	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7608	pCtx->cr4 = u64Val;
7609	}
7610
7611	if (fWhat & CPUMCTX_EXTRN_CR3)
7612	{
7613	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7614	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7615	\|\| ( pVM->hm.s.fNestedPaging
7616	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7617	{
7618	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7619	VMXLOCAL_BREAK_RC(rc);
7620	if (pCtx->cr3 != u64Val)
7621	{
7622	pCtx->cr3 = u64Val;
7623	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7624	}
7625
7626	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7627	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7628	if (CPUMIsGuestInPAEModeEx(pCtx))
7629	{
7630	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7631	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7632	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7633	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7634	VMXLOCAL_BREAK_RC(rc);
7635	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7636	}
7637	}
7638	}
7639
7640	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7641	# if 0
7642	/** @todo NSTVMX: We handle each of these fields individually by passing it to IEM
7643	* VM-exit handlers. We might handle it differently when using the fast path. */
7644	/*
7645	* The hardware virtualization state currently consists of VMCS fields that may be
7646	* modified by execution of the nested-guest (that are not part of the general
7647	* guest state) and is visible to guest software. Hence, it is technically part of
7648	* the guest-CPU state when executing a nested-guest.
7649	*/
7650	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7651	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7652	{
7653	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7654	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7655	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7656	VMXLOCAL_BREAK_RC(rc);
7657
7658	/*
7659	* VM-entry can fail due to invalid-guest state, machine-check events and
7660	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7661	* all other VMCS fields are left unmodified on VM-entry failure.
7662	*
7663	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7664	*/
7665	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7666	if (!fEntryFailed)
7667	{
7668	/*
7669	* Some notes on VMCS fields that may need importing when the fast path
7670	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7671	*
7672	* Requires fixing up when using hardware-assisted VMX:
7673	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7674	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7675	* - IDT-vectoring info: Think about this.
7676	* - IDT-vectoring error code: Think about this.
7677	*
7678	* Emulated:
7679	* - Guest-interruptiblity state: Derived from FFs and RIP.
7680	* - Guest pending debug exceptions: Derived from DR6.
7681	* - Guest activity state: Emulated from EM state.
7682	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7683	* - Entry-interrupt info: Emulated, cleared to 0.
7684	*/
7685	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7686	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7687	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7688	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7689	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7690	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7691	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7692	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7693	/** @todo NSTVMX: Save and adjust preemption timer value. */
7694	}
7695
7696	VMXLOCAL_BREAK_RC(rc);
7697	}
7698	# endif
7699	#endif
7700	}
7701	} while (0);
7702
7703	if (RT_SUCCESS(rc))
7704	{
7705	/* Update fExtrn. */
7706	pCtx->fExtrn &= ~fWhat;
7707
7708	/* If everything has been imported, clear the HM keeper bit. */
7709	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7710	{
7711	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7712	Assert(!pCtx->fExtrn);
7713	}
7714	}
7715	}
7716	else
7717	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7718
7719	ASMSetFlags(fEFlags);
7720
7721	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7722
7723	if (RT_SUCCESS(rc))
7724	{ /* likely */ }
7725	else
7726	return rc;
7727
7728	/*
7729	* Honor any pending CR3 updates.
7730	*
7731	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7732	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7733	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7734	*
7735	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7736	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7737	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7738	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7739	*
7740	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7741	*/
7742	if (VMMRZCallRing3IsEnabled(pVCpu))
7743	{
7744	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7745	{
7746	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7747	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7748	}
7749
7750	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7751	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7752
7753	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7754	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7755	}
7756
7757	return VINF_SUCCESS;
7758	#undef VMXLOCAL_BREAK_RC
7759	}
7760
7761
7762	/**
7763	* Saves the guest state from the VMCS into the guest-CPU context.
7764	*
7765	* @returns VBox status code.
7766	* @param pVCpu The cross context virtual CPU structure.
7767	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7768	*/
7769	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7770	{
7771	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7772	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7773	}
7774
7775
7776	/**
7777	* Check per-VM and per-VCPU force flag actions that require us to go back to
7778	* ring-3 for one reason or another.
7779	*
7780	* @returns Strict VBox status code (i.e. informational status codes too)
7781	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7782	* ring-3.
7783	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7784	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7785	* interrupts)
7786	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7787	* all EMTs to be in ring-3.
7788	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7789	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7790	* to the EM loop.
7791	*
7792	* @param pVCpu The cross context virtual CPU structure.
7793	* @param fStepping Whether we are single-stepping the guest using the
7794	* hypervisor debugger.
7795	*/
7796	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7797	{
7798	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7799
7800	/*
7801	* Update pending interrupts into the APIC's IRR.
7802	*/
7803	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7804	APICUpdatePendingInterrupts(pVCpu);
7805
7806	/*
7807	* Anything pending? Should be more likely than not if we're doing a good job.
7808	*/
7809	PVM pVM = pVCpu->CTX_SUFF(pVM);
7810	if ( !fStepping
7811	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7812	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7813	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7814	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7815	return VINF_SUCCESS;
7816
7817	/* Pending PGM C3 sync. */
7818	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7819	{
7820	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7821	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7822	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7823	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7824	if (rcStrict2 != VINF_SUCCESS)
7825	{
7826	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7827	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7828	return rcStrict2;
7829	}
7830	}
7831
7832	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7833	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7834	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7835	{
7836	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7837	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7838	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7839	return rc2;
7840	}
7841
7842	/* Pending VM request packets, such as hardware interrupts. */
7843	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7844	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7845	{
7846	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7847	return VINF_EM_PENDING_REQUEST;
7848	}
7849
7850	/* Pending PGM pool flushes. */
7851	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7852	{
7853	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7854	return VINF_PGM_POOL_FLUSH_PENDING;
7855	}
7856
7857	/* Pending DMA requests. */
7858	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7859	{
7860	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7861	return VINF_EM_RAW_TO_R3;
7862	}
7863
7864	return VINF_SUCCESS;
7865	}
7866
7867
7868	/**
7869	* Converts any TRPM trap into a pending HM event. This is typically used when
7870	* entering from ring-3 (not longjmp returns).
7871	*
7872	* @param pVCpu The cross context virtual CPU structure.
7873	*/
7874	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7875	{
7876	Assert(TRPMHasTrap(pVCpu));
7877	Assert(!pVCpu->hm.s.Event.fPending);
7878
7879	uint8_t uVector;
7880	TRPMEVENT enmTrpmEvent;
7881	RTGCUINT uErrCode;
7882	RTGCUINTPTR GCPtrFaultAddress;
7883	uint8_t cbInstr;
7884
7885	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7886	AssertRC(rc);
7887
7888	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7889	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7890	if (enmTrpmEvent == TRPM_TRAP)
7891	{
7892	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7893	* generated using INT1 (ICEBP). */
7894	switch (uVector)
7895	{
7896	case X86_XCPT_NMI:
7897	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7898	break;
7899
7900	case X86_XCPT_BP:
7901	case X86_XCPT_OF:
7902	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7903	break;
7904
7905	case X86_XCPT_PF:
7906	case X86_XCPT_DF:
7907	case X86_XCPT_TS:
7908	case X86_XCPT_NP:
7909	case X86_XCPT_SS:
7910	case X86_XCPT_GP:
7911	case X86_XCPT_AC:
7912	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7913	RT_FALL_THRU();
7914	default:
7915	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7916	break;
7917	}
7918	}
7919	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7920	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7921	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7922	{
7923	switch (uVector)
7924	{
7925	case X86_XCPT_BP:
7926	case X86_XCPT_OF:
7927	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7928	break;
7929
7930	default:
7931	Assert(uVector == X86_XCPT_DB);
7932	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7933	break;
7934	}
7935	}
7936	else
7937	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7938
7939	rc = TRPMResetTrap(pVCpu);
7940	AssertRC(rc);
7941	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7942	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7943
7944	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7945	}
7946
7947
7948	/**
7949	* Converts the pending HM event into a TRPM trap.
7950	*
7951	* @param pVCpu The cross context virtual CPU structure.
7952	*/
7953	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7954	{
7955	Assert(pVCpu->hm.s.Event.fPending);
7956
7957	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7958	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7959	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7960	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7961
7962	/* If a trap was already pending, we did something wrong! */
7963	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7964
7965	/** @todo Use HMVmxEventToTrpmEventType() later. */
7966	TRPMEVENT enmTrapType;
7967	switch (uVectorType)
7968	{
7969	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7970	enmTrapType = TRPM_HARDWARE_INT;
7971	break;
7972
7973	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7974	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7975	enmTrapType = TRPM_TRAP;
7976	break;
7977
7978	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7979	Assert(uVector == X86_XCPT_DB);
7980	enmTrapType = TRPM_SOFTWARE_INT;
7981	break;
7982
7983	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7984	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7985	enmTrapType = TRPM_SOFTWARE_INT;
7986	break;
7987
7988	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7989	enmTrapType = TRPM_SOFTWARE_INT;
7990	break;
7991
7992	default:
7993	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7994	enmTrapType = TRPM_32BIT_HACK;
7995	break;
7996	}
7997
7998	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7999
8000	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8001	AssertRC(rc);
8002
8003	if (fErrorCodeValid)
8004	TRPMSetErrorCode(pVCpu, uErrorCode);
8005
8006	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8007	&& uVector == X86_XCPT_PF)
8008	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8009	else if (enmTrapType == TRPM_SOFTWARE_INT)
8010	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8011
8012	/* We're now done converting the pending event. */
8013	pVCpu->hm.s.Event.fPending = false;
8014	}
8015
8016
8017	/**
8018	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8019	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8020	*
8021	* @param pVCpu The cross context virtual CPU structure.
8022	* @param pVmcsInfo The VMCS info. object.
8023	*/
8024	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8025	{
8026	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8027	{
8028	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8029	{
8030	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8031	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8032	AssertRC(rc);
8033	}
8034	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8035	}
8036
8037
8038	/**
8039	* Clears the interrupt-window exiting control in the VMCS.
8040	*
8041	* @param pVmcsInfo The VMCS info. object.
8042	*/
8043	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8044	{
8045	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8046	{
8047	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8048	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8049	}
8050	return VINF_SUCCESS;
8051	}
8052
8053
8054	/**
8055	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8056	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8057	*
8058	* @param pVCpu The cross context virtual CPU structure.
8059	* @param pVmcsInfo The VMCS info. object.
8060	*/
8061	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8062	{
8063	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8064	{
8065	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8066	{
8067	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8068	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8069	AssertRC(rc);
8070	Log4Func(("Setup NMI-window exiting\n"));
8071	}
8072	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8073	}
8074
8075
8076	/**
8077	* Clears the NMI-window exiting control in the VMCS.
8078	*
8079	* @param pVmcsInfo The VMCS info. object.
8080	*/
8081	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8082	{
8083	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8084	{
8085	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8086	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8087	}
8088	return VINF_SUCCESS;
8089	}
8090
8091
8092	/**
8093	* Does the necessary state syncing before returning to ring-3 for any reason
8094	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8095	*
8096	* @returns VBox status code.
8097	* @param pVCpu The cross context virtual CPU structure.
8098	* @param fImportState Whether to import the guest state from the VMCS back
8099	* to the guest-CPU context.
8100	*
8101	* @remarks No-long-jmp zone!!!
8102	*/
8103	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8104	{
8105	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8106	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8107
8108	RTCPUID idCpu = RTMpCpuId();
8109	Log4Func(("HostCpuId=%u\n", idCpu));
8110
8111	/*
8112	* !!! IMPORTANT !!!
8113	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8114	*/
8115
8116	/* Save the guest state if necessary. */
8117	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8118	if (fImportState)
8119	{
8120	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8121	AssertRCReturn(rc, rc);
8122	}
8123
8124	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8125	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8126	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8127
8128	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8129	#ifdef VBOX_STRICT
8130	if (CPUMIsHyperDebugStateActive(pVCpu))
8131	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8132	#endif
8133	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8134	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8135	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8136
8137	#if HC_ARCH_BITS == 64
8138	/* Restore host-state bits that VT-x only restores partially. */
8139	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8140	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8141	{
8142	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8143	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8144	}
8145	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8146	#endif
8147
8148	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8149	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8150	{
8151	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8152	if (!fImportState)
8153	{
8154	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8155	AssertRCReturn(rc, rc);
8156	}
8157	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8158	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8159	}
8160	else
8161	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8162
8163	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8164	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8165
8166	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8167	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8168	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8169	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8170	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8171	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8172	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8173	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8174	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8175
8176	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8177
8178	/** @todo This partially defeats the purpose of having preemption hooks.
8179	* The problem is, deregistering the hooks should be moved to a place that
8180	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8181	* context.
8182	*/
8183	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8184	AssertRCReturn(rc, rc);
8185
8186	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8187	NOREF(idCpu);
8188	return VINF_SUCCESS;
8189	}
8190
8191
8192	/**
8193	* Leaves the VT-x session.
8194	*
8195	* @returns VBox status code.
8196	* @param pVCpu The cross context virtual CPU structure.
8197	*
8198	* @remarks No-long-jmp zone!!!
8199	*/
8200	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8201	{
8202	HM_DISABLE_PREEMPT(pVCpu);
8203	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8204	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8205	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8206
8207	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8208	and done this from the VMXR0ThreadCtxCallback(). */
8209	if (!pVCpu->hm.s.fLeaveDone)
8210	{
8211	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8212	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8213	pVCpu->hm.s.fLeaveDone = true;
8214	}
8215	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8216
8217	/*
8218	* !!! IMPORTANT !!!
8219	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8220	*/
8221
8222	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8223	/** @todo Deregistering here means we need to VMCLEAR always
8224	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8225	* for calling VMMR0ThreadCtxHookDisable here! */
8226	VMMR0ThreadCtxHookDisable(pVCpu);
8227
8228	/* Leave HM context. This takes care of local init (term). */
8229	int rc = HMR0LeaveCpu(pVCpu);
8230
8231	HM_RESTORE_PREEMPT();
8232	return rc;
8233	}
8234
8235
8236	/**
8237	* Does the necessary state syncing before doing a longjmp to ring-3.
8238	*
8239	* @returns VBox status code.
8240	* @param pVCpu The cross context virtual CPU structure.
8241	*
8242	* @remarks No-long-jmp zone!!!
8243	*/
8244	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8245	{
8246	return hmR0VmxLeaveSession(pVCpu);
8247	}
8248
8249
8250	/**
8251	* Take necessary actions before going back to ring-3.
8252	*
8253	* An action requires us to go back to ring-3. This function does the necessary
8254	* steps before we can safely return to ring-3. This is not the same as longjmps
8255	* to ring-3, this is voluntary and prepares the guest so it may continue
8256	* executing outside HM (recompiler/IEM).
8257	*
8258	* @returns VBox status code.
8259	* @param pVCpu The cross context virtual CPU structure.
8260	* @param rcExit The reason for exiting to ring-3. Can be
8261	* VINF_VMM_UNKNOWN_RING3_CALL.
8262	*/
8263	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8264	{
8265	Assert(pVCpu);
8266	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8267
8268	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8269	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8270	{
8271	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8272	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8273	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8274	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8275	}
8276
8277	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8278	VMMRZCallRing3Disable(pVCpu);
8279	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8280
8281	/*
8282	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8283	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8284	*
8285	* This is because execution may continue from ring-3 and we would need to inject
8286	* the event from there (hence place it back in TRPM).
8287	*/
8288	if (pVCpu->hm.s.Event.fPending)
8289	{
8290	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8291	Assert(!pVCpu->hm.s.Event.fPending);
8292
8293	/* Clear the events from the VMCS. */
8294	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8295	AssertRCReturn(rc, rc);
8296	}
8297	#ifdef VBOX_STRICT
8298	else
8299	{
8300	/*
8301	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8302	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8303	* occasionally, see @bugref{9180#c42}.
8304	*/
8305	uint32_t uEntryIntInfo;
8306	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8307	AssertRC(rc);
8308	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8309	}
8310	#endif
8311
8312	/*
8313	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8314	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8315	* (e.g. TPR below threshold).
8316	*/
8317	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8318	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8319	AssertRCReturn(rc, rc);
8320
8321	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8322	and if we're injecting an event we should have a TRPM trap pending. */
8323	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8324	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8325	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8326	#endif
8327
8328	/* Save guest state and restore host state bits. */
8329	rc = hmR0VmxLeaveSession(pVCpu);
8330	AssertRCReturn(rc, rc);
8331	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8332
8333	/* Thread-context hooks are unregistered at this point!!! */
8334
8335	/* Sync recompiler state. */
8336	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8337	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8338	\| CPUM_CHANGED_LDTR
8339	\| CPUM_CHANGED_GDTR
8340	\| CPUM_CHANGED_IDTR
8341	\| CPUM_CHANGED_TR
8342	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8343	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8344	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8345	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8346
8347	Assert(!pVCpu->hm.s.fClearTrapFlag);
8348
8349	/* Update the exit-to-ring 3 reason. */
8350	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8351
8352	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8353	if ( rcExit != VINF_EM_RAW_INTERRUPT
8354	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8355	{
8356	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8357	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8358	}
8359
8360	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8361
8362	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8363	VMMRZCallRing3RemoveNotification(pVCpu);
8364	VMMRZCallRing3Enable(pVCpu);
8365
8366	return rc;
8367	}
8368
8369
8370	/**
8371	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8372	* longjump to ring-3 and possibly get preempted.
8373	*
8374	* @returns VBox status code.
8375	* @param pVCpu The cross context virtual CPU structure.
8376	* @param enmOperation The operation causing the ring-3 longjump.
8377	* @param pvUser User argument, currently unused, NULL.
8378	*/
8379	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8380	{
8381	RT_NOREF(pvUser);
8382	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8383	{
8384	/*
8385	* !!! IMPORTANT !!!
8386	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8387	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8388	*/
8389	VMMRZCallRing3RemoveNotification(pVCpu);
8390	VMMRZCallRing3Disable(pVCpu);
8391	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8392	RTThreadPreemptDisable(&PreemptState);
8393
8394	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8395	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8396	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8397	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8398
8399	#if HC_ARCH_BITS == 64
8400	/* Restore host-state bits that VT-x only restores partially. */
8401	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8402	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8403	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8404	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8405	#endif
8406
8407	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8408	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8409	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8410
8411	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8412	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8413	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8414
8415	/* Clear the current VMCS data back to memory. */
8416	hmR0VmxClearVmcs(pVmcsInfo);
8417
8418	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8419	VMMR0ThreadCtxHookDisable(pVCpu);
8420	HMR0LeaveCpu(pVCpu);
8421	RTThreadPreemptRestore(&PreemptState);
8422	return VINF_SUCCESS;
8423	}
8424
8425	Assert(pVCpu);
8426	Assert(pvUser);
8427	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8428	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8429
8430	VMMRZCallRing3Disable(pVCpu);
8431	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8432
8433	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8434
8435	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8436	AssertRCReturn(rc, rc);
8437
8438	VMMRZCallRing3Enable(pVCpu);
8439	return VINF_SUCCESS;
8440	}
8441
8442
8443	/**
8444	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8445	* stack.
8446	*
8447	* @returns Strict VBox status code (i.e. informational status codes too).
8448	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8449	* @param pVCpu The cross context virtual CPU structure.
8450	* @param uValue The value to push to the guest stack.
8451	*/
8452	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8453	{
8454	/*
8455	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8456	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8457	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8458	*/
8459	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8460	if (pCtx->sp == 1)
8461	return VINF_EM_RESET;
8462	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8463	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8464	AssertRC(rc);
8465	return rc;
8466	}
8467
8468
8469	/**
8470	* Injects an event into the guest upon VM-entry by updating the relevant fields
8471	* in the VM-entry area in the VMCS.
8472	*
8473	* @returns Strict VBox status code (i.e. informational status codes too).
8474	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8475	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8476	*
8477	* @param pVCpu The cross context virtual CPU structure.
8478	* @param pVmxTransient The VMX-transient structure.
8479	* @param pEvent The event being injected.
8480	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8481	* This will be updated if necessary. This cannot not
8482	* be NULL.
8483	* @param fStepping Whether we're single-stepping guest execution and
8484	* should return VINF_EM_DBG_STEPPED if the event is
8485	* injected directly (registers modified by us, not by
8486	* hardware on VM-entry).
8487	*/
8488	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8489	uint32_t *pfIntrState)
8490	{
8491	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8492	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8493	Assert(pfIntrState);
8494
8495	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8496	uint32_t u32IntInfo = pEvent->u64IntInfo;
8497	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8498	uint32_t const cbInstr = pEvent->cbInstr;
8499	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8500	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8501	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8502
8503	#ifdef VBOX_STRICT
8504	/*
8505	* Validate the error-code-valid bit for hardware exceptions.
8506	* No error codes for exceptions in real-mode.
8507	*
8508	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8509	*/
8510	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8511	&& !CPUMIsGuestInRealModeEx(pCtx))
8512	{
8513	switch (uVector)
8514	{
8515	case X86_XCPT_PF:
8516	case X86_XCPT_DF:
8517	case X86_XCPT_TS:
8518	case X86_XCPT_NP:
8519	case X86_XCPT_SS:
8520	case X86_XCPT_GP:
8521	case X86_XCPT_AC:
8522	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8523	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8524	RT_FALL_THRU();
8525	default:
8526	break;
8527	}
8528	}
8529
8530	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8531	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8532	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8533	#endif
8534
8535	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8536
8537	/*
8538	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8539	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8540	* interrupt handler in the (real-mode) guest.
8541	*
8542	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8543	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8544	*/
8545	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8546	{
8547	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8548	{
8549	/*
8550	* For CPUs with unrestricted guest execution enabled and with the guest
8551	* in real-mode, we must not set the deliver-error-code bit.
8552	*
8553	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8554	*/
8555	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8556	}
8557	else
8558	{
8559	PVM pVM = pVCpu->CTX_SUFF(pVM);
8560	Assert(PDMVmmDevHeapIsEnabled(pVM));
8561	Assert(pVM->hm.s.vmx.pRealModeTSS);
8562	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8563
8564	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8565	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8566	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8567	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8568	AssertRCReturn(rc2, rc2);
8569
8570	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8571	size_t const cbIdtEntry = sizeof(X86IDTR16);
8572	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8573	{
8574	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8575	if (uVector == X86_XCPT_DF)
8576	return VINF_EM_RESET;
8577
8578	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8579	No error codes for exceptions in real-mode. */
8580	if (uVector == X86_XCPT_GP)
8581	{
8582	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8583	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8584	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8585	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8586	HMEVENT EventXcptDf;
8587	RT_ZERO(EventXcptDf);
8588	EventXcptDf.u64IntInfo = uXcptDfInfo;
8589	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8590	}
8591
8592	/*
8593	* If we're injecting an event with no valid IDT entry, inject a #GP.
8594	* No error codes for exceptions in real-mode.
8595	*
8596	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8597	*/
8598	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8599	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8600	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8601	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8602	HMEVENT EventXcptGp;
8603	RT_ZERO(EventXcptGp);
8604	EventXcptGp.u64IntInfo = uXcptGpInfo;
8605	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8606	}
8607
8608	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8609	uint16_t uGuestIp = pCtx->ip;
8610	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8611	{
8612	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8613	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8614	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8615	}
8616	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8617	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8618
8619	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8620	X86IDTR16 IdtEntry;
8621	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8622	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8623	AssertRCReturn(rc2, rc2);
8624
8625	/* Construct the stack frame for the interrupt/exception handler. */
8626	VBOXSTRICTRC rcStrict;
8627	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8628	if (rcStrict == VINF_SUCCESS)
8629	{
8630	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8631	if (rcStrict == VINF_SUCCESS)
8632	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8633	}
8634
8635	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8636	if (rcStrict == VINF_SUCCESS)
8637	{
8638	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8639	pCtx->rip = IdtEntry.offSel;
8640	pCtx->cs.Sel = IdtEntry.uSel;
8641	pCtx->cs.ValidSel = IdtEntry.uSel;
8642	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8643	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8644	&& uVector == X86_XCPT_PF)
8645	pCtx->cr2 = GCPtrFault;
8646
8647	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8648	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8649	\| HM_CHANGED_GUEST_RSP);
8650
8651	/*
8652	* If we delivered a hardware exception (other than an NMI) and if there was
8653	* block-by-STI in effect, we should clear it.
8654	*/
8655	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8656	{
8657	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8658	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8659	Log4Func(("Clearing inhibition due to STI\n"));
8660	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8661	}
8662
8663	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8664	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8665
8666	/*
8667	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8668	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8669	*/
8670	pVCpu->hm.s.Event.fPending = false;
8671
8672	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8673	if (fStepping)
8674	rcStrict = VINF_EM_DBG_STEPPED;
8675	}
8676	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8677	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8678	return rcStrict;
8679	}
8680	}
8681
8682	/*
8683	* Validate.
8684	*/
8685	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8686	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8687
8688	/*
8689	* Inject the event into the VMCS.
8690	*/
8691	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8692	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8693	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8694	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8695	AssertRCReturn(rc, rc);
8696
8697	/*
8698	* Update guest CR2 if this is a page-fault.
8699	*/
8700	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8701	&& uVector == X86_XCPT_PF)
8702	pCtx->cr2 = GCPtrFault;
8703
8704	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8705	return VINF_SUCCESS;
8706	}
8707
8708
8709	/**
8710	* Evaluates the event to be delivered to the guest and sets it as the pending
8711	* event.
8712	*
8713	* @returns Strict VBox status code (i.e. informational status codes too).
8714	* @param pVCpu The cross context virtual CPU structure.
8715	* @param pVmxTransient The VMX-transient structure.
8716	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8717	*/
8718	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8719	{
8720	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8721	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8722
8723	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8724	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8725	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8726	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8727	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8728
8729	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8730	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8731	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8732	Assert(!TRPMHasTrap(pVCpu));
8733	Assert(pfIntrState);
8734
8735	*pfIntrState = fIntrState;
8736
8737	/*
8738	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8739	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8740	*/
8741	/** @todo SMI. SMIs take priority over NMIs. */
8742	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8743	{
8744	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8745	if ( !pVCpu->hm.s.Event.fPending
8746	&& !fBlockNmi
8747	&& !fBlockSti
8748	&& !fBlockMovSS)
8749	{
8750	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8751	if ( pVmxTransient->fIsNestedGuest
8752	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8753	return IEMExecVmxVmexitNmi(pVCpu);
8754	#endif
8755	hmR0VmxSetPendingXcptNmi(pVCpu);
8756	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8757	Log4Func(("Pending NMI\n"));
8758	}
8759	else
8760	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8761	}
8762	/*
8763	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8764	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8765	*/
8766	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8767	&& !pVCpu->hm.s.fSingleInstruction)
8768	{
8769	Assert(!DBGFIsStepping(pVCpu));
8770	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8771	AssertRCReturn(rc, rc);
8772	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8773	if ( !pVCpu->hm.s.Event.fPending
8774	&& !fBlockInt
8775	&& !fBlockSti
8776	&& !fBlockMovSS)
8777	{
8778	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8779	if ( pVmxTransient->fIsNestedGuest
8780	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8781	{
8782	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8783	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8784	return rcStrict;
8785	}
8786	#endif
8787	uint8_t u8Interrupt;
8788	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8789	if (RT_SUCCESS(rc))
8790	{
8791	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8792	if ( pVmxTransient->fIsNestedGuest
8793	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8794	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8795	{
8796	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8797	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8798	return rcStrict;
8799	}
8800	#endif
8801	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8802	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8803	}
8804	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8805	{
8806	if ( !pVmxTransient->fIsNestedGuest
8807	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8808	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8809	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8810
8811	/*
8812	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8813	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8814	* need to re-set this force-flag here.
8815	*/
8816	}
8817	else
8818	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8819	}
8820	else
8821	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8822	}
8823
8824	return VINF_SUCCESS;
8825	}
8826
8827
8828	/**
8829	* Injects any pending events into the guest if the guest is in a state to
8830	* receive them.
8831	*
8832	* @returns Strict VBox status code (i.e. informational status codes too).
8833	* @param pVCpu The cross context virtual CPU structure.
8834	* @param pVmxTransient The VMX-transient structure.
8835	* @param fIntrState The VT-x guest-interruptibility state.
8836	* @param fStepping Whether we are single-stepping the guest using the
8837	* hypervisor debugger and should return
8838	* VINF_EM_DBG_STEPPED if the event was dispatched
8839	* directly.
8840	*/
8841	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8842	{
8843	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8844	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8845
8846	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8847	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8848
8849	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8850	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8851	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8852	Assert(!TRPMHasTrap(pVCpu));
8853
8854	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8855	if (pVCpu->hm.s.Event.fPending)
8856	{
8857	/*
8858	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8859	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8860	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8861	*
8862	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8863	*/
8864	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8865	#ifdef VBOX_STRICT
8866	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8867	{
8868	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8869	Assert(!fBlockInt);
8870	Assert(!fBlockSti);
8871	Assert(!fBlockMovSS);
8872	}
8873	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8874	{
8875	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8876	Assert(!fBlockSti);
8877	Assert(!fBlockMovSS);
8878	Assert(!fBlockNmi);
8879	}
8880	#endif
8881	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8882	uIntType));
8883
8884	/*
8885	* Inject the event and get any changes to the guest-interruptibility state.
8886	*
8887	* The guest-interruptibility state may need to be updated if we inject the event
8888	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8889	*/
8890	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8891	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8892
8893	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8894	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8895	else
8896	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8897	}
8898
8899	/*
8900	* Update the guest-interruptibility state.
8901	*
8902	* This is required for the real-on-v86 software interrupt injection case above, as well as
8903	* updates to the guest state from ring-3 or IEM/REM.
8904	*/
8905	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8906	AssertRCReturn(rc, rc);
8907
8908	/*
8909	* There's no need to clear the VM-entry interruption-information field here if we're not
8910	* injecting anything. VT-x clears the valid bit on every VM-exit.
8911	*
8912	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8913	*/
8914
8915	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8916	NOREF(fBlockMovSS); NOREF(fBlockSti);
8917	return rcStrict;
8918	}
8919
8920
8921	/**
8922	* Enters the VT-x session.
8923	*
8924	* @returns VBox status code.
8925	* @param pVCpu The cross context virtual CPU structure.
8926	*/
8927	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8928	{
8929	AssertPtr(pVCpu);
8930	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8931	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8932
8933	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8934	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8935	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8936
8937	#ifdef VBOX_STRICT
8938	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8939	RTCCUINTREG uHostCR4 = ASMGetCR4();
8940	if (!(uHostCR4 & X86_CR4_VMXE))
8941	{
8942	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8943	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8944	}
8945	#endif
8946
8947	/*
8948	* Load the appropriate VMCS as the current and active one.
8949	*/
8950	PVMXVMCSINFO pVmcsInfo;
8951	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8952	if (!fInNestedGuestMode)
8953	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8954	else
8955	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8956	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8957	if (RT_SUCCESS(rc))
8958	{
8959	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8960	pVCpu->hm.s.fLeaveDone = false;
8961	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8962
8963	/*
8964	* Do the EMT scheduled L1D flush here if needed.
8965	*/
8966	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8967	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8968	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8969	hmR0MdsClear();
8970	}
8971	return rc;
8972	}
8973
8974
8975	/**
8976	* The thread-context callback (only on platforms which support it).
8977	*
8978	* @param enmEvent The thread-context event.
8979	* @param pVCpu The cross context virtual CPU structure.
8980	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8981	* @thread EMT(pVCpu)
8982	*/
8983	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8984	{
8985	NOREF(fGlobalInit);
8986
8987	switch (enmEvent)
8988	{
8989	case RTTHREADCTXEVENT_OUT:
8990	{
8991	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8992	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8993	VMCPU_ASSERT_EMT(pVCpu);
8994
8995	/* No longjmps (logger flushes, locks) in this fragile context. */
8996	VMMRZCallRing3Disable(pVCpu);
8997	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8998
8999	/* Restore host-state (FPU, debug etc.) */
9000	if (!pVCpu->hm.s.fLeaveDone)
9001	{
9002	/*
9003	* Do -not- import the guest-state here as we might already be in the middle of importing
9004	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9005	*/
9006	hmR0VmxLeave(pVCpu, false /* fImportState */);
9007	pVCpu->hm.s.fLeaveDone = true;
9008	}
9009
9010	/* Leave HM context, takes care of local init (term). */
9011	int rc = HMR0LeaveCpu(pVCpu);
9012	AssertRC(rc);
9013
9014	/* Restore longjmp state. */
9015	VMMRZCallRing3Enable(pVCpu);
9016	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9017	break;
9018	}
9019
9020	case RTTHREADCTXEVENT_IN:
9021	{
9022	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9023	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9024	VMCPU_ASSERT_EMT(pVCpu);
9025
9026	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9027	VMMRZCallRing3Disable(pVCpu);
9028	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9029
9030	/* Initialize the bare minimum state required for HM. This takes care of
9031	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9032	int rc = hmR0EnterCpu(pVCpu);
9033	AssertRC(rc);
9034	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9035	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9036
9037	/* Load the active VMCS as the current one. */
9038	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9039	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9040	AssertRC(rc);
9041	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9042	pVCpu->hm.s.fLeaveDone = false;
9043
9044	/* Do the EMT scheduled L1D flush if needed. */
9045	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9046	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9047
9048	/* Restore longjmp state. */
9049	VMMRZCallRing3Enable(pVCpu);
9050	break;
9051	}
9052
9053	default:
9054	break;
9055	}
9056	}
9057
9058
9059	/**
9060	* Exports the host state into the VMCS host-state area.
9061	* Sets up the VM-exit MSR-load area.
9062	*
9063	* The CPU state will be loaded from these fields on every successful VM-exit.
9064	*
9065	* @returns VBox status code.
9066	* @param pVCpu The cross context virtual CPU structure.
9067	*
9068	* @remarks No-long-jump zone!!!
9069	*/
9070	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9071	{
9072	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9073
9074	int rc = VINF_SUCCESS;
9075	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9076	{
9077	rc = hmR0VmxExportHostControlRegs();
9078	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9079
9080	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9081	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9082
9083	rc = hmR0VmxExportHostMsrs(pVCpu);
9084	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9085
9086	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9087	}
9088	return rc;
9089	}
9090
9091
9092	/**
9093	* Saves the host state in the VMCS host-state.
9094	*
9095	* @returns VBox status code.
9096	* @param pVCpu The cross context virtual CPU structure.
9097	*
9098	* @remarks No-long-jump zone!!!
9099	*/
9100	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9101	{
9102	AssertPtr(pVCpu);
9103	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9104
9105	/*
9106	* Export the host state here while entering HM context.
9107	* When thread-context hooks are used, we might get preempted and have to re-save the host
9108	* state but most of the time we won't be, so do it here before we disable interrupts.
9109	*/
9110	return hmR0VmxExportHostState(pVCpu);
9111	}
9112
9113
9114	/**
9115	* Exports the guest state into the VMCS guest-state area.
9116	*
9117	* The will typically be done before VM-entry when the guest-CPU state and the
9118	* VMCS state may potentially be out of sync.
9119	*
9120	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9121	* VM-entry controls.
9122	* Sets up the appropriate VMX non-root function to execute guest code based on
9123	* the guest CPU mode.
9124	*
9125	* @returns VBox strict status code.
9126	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9127	* without unrestricted guest execution and the VMMDev is not presently
9128	* mapped (e.g. EFI32).
9129	*
9130	* @param pVCpu The cross context virtual CPU structure.
9131	* @param pVmxTransient The VMX-transient structure.
9132	*
9133	* @remarks No-long-jump zone!!!
9134	*/
9135	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9136	{
9137	AssertPtr(pVCpu);
9138	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9139	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9140
9141	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9142
9143	/*
9144	* Determine real-on-v86 mode.
9145	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9146	*/
9147	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9148	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9149	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9150	pVmcsInfo->RealMode. fRealOnV86Active = false;
9151	else
9152	{
9153	Assert(!pVmxTransient->fIsNestedGuest);
9154	pVmcsInfo->RealMode.fRealOnV86Active = true;
9155	}
9156
9157	/*
9158	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9159	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9160	*/
9161	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9162	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9163	* be a need to evaluate this everytime since I'm pretty sure we intercept
9164	* all guest paging mode changes. */
9165	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9166	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9167
9168	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9169	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9170
9171	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9172	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9173
9174	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9175	if (rcStrict == VINF_SUCCESS)
9176	{ /* likely */ }
9177	else
9178	{
9179	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9180	return rcStrict;
9181	}
9182
9183	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9184	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9185
9186	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9187	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9188
9189	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9190	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9191
9192	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9193	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9194
9195	rc = hmR0VmxExportGuestRip(pVCpu);
9196	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9197	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9198	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9199
9200	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9201	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9202	\| HM_CHANGED_GUEST_CR2
9203	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9204	\| HM_CHANGED_GUEST_X87
9205	\| HM_CHANGED_GUEST_SSE_AVX
9206	\| HM_CHANGED_GUEST_OTHER_XSAVE
9207	\| HM_CHANGED_GUEST_XCRx
9208	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9209	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9210	\| HM_CHANGED_GUEST_TSC_AUX
9211	\| HM_CHANGED_GUEST_OTHER_MSRS
9212	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9213	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9214
9215	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9216	return rc;
9217	}
9218
9219
9220	/**
9221	* Exports the state shared between the host and guest into the VMCS.
9222	*
9223	* @param pVCpu The cross context virtual CPU structure.
9224	* @param pVmxTransient The VMX-transient structure.
9225	*
9226	* @remarks No-long-jump zone!!!
9227	*/
9228	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9229	{
9230	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9231	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9232
9233	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9234	{
9235	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9236	AssertRC(rc);
9237	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9238
9239	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9240	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9241	{
9242	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9243	AssertRC(rc);
9244	}
9245	}
9246
9247	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9248	{
9249	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9250	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9251	}
9252
9253	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9254	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9255	}
9256
9257
9258	/**
9259	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9260	*
9261	* @returns Strict VBox status code (i.e. informational status codes too).
9262	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9263	* without unrestricted guest execution and the VMMDev is not presently
9264	* mapped (e.g. EFI32).
9265	*
9266	* @param pVCpu The cross context virtual CPU structure.
9267	* @param pVmxTransient The VMX-transient structure.
9268	*
9269	* @remarks No-long-jump zone!!!
9270	*/
9271	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9272	{
9273	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9274	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9275	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9276
9277	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9278	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9279	#endif
9280
9281	/*
9282	* For many exits it's only RIP that changes and hence try to export it first
9283	* without going through a lot of change flag checks.
9284	*/
9285	VBOXSTRICTRC rcStrict;
9286	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9287	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9288	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9289	{
9290	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9291	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9292	{ /* likely */}
9293	else
9294	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9296	}
9297	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9298	{
9299	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9300	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9301	{ /* likely */}
9302	else
9303	{
9304	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9305	VBOXSTRICTRC_VAL(rcStrict)));
9306	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9307	return rcStrict;
9308	}
9309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9310	}
9311	else
9312	rcStrict = VINF_SUCCESS;
9313
9314	#ifdef VBOX_STRICT
9315	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9316	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9317	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9318	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9319	("fCtxChanged=%#RX64\n", fCtxChanged));
9320	#endif
9321	return rcStrict;
9322	}
9323
9324
9325	/**
9326	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9327	* and update error record fields accordingly.
9328	*
9329	* @return VMX_IGS_* return codes.
9330	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9331	* wrong with the guest state.
9332	*
9333	* @param pVCpu The cross context virtual CPU structure.
9334	* @param pVmcsInfo The VMCS info. object.
9335	*
9336	* @remarks This function assumes our cache of the VMCS controls
9337	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9338	*/
9339	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9340	{
9341	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9342	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9343	uError = (err); \
9344	break; \
9345	} else do { } while (0)
9346
9347	int rc;
9348	PVM pVM = pVCpu->CTX_SUFF(pVM);
9349	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9350	uint32_t uError = VMX_IGS_ERROR;
9351	uint32_t u32Val;
9352	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9353
9354	do
9355	{
9356	/*
9357	* CR0.
9358	*/
9359	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9360	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9361	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9362	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9363	if (fUnrestrictedGuest)
9364	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9365
9366	uint32_t u32GuestCr0;
9367	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9368	AssertRCBreak(rc);
9369	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9370	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9371	if ( !fUnrestrictedGuest
9372	&& (u32GuestCr0 & X86_CR0_PG)
9373	&& !(u32GuestCr0 & X86_CR0_PE))
9374	{
9375	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9376	}
9377
9378	/*
9379	* CR4.
9380	*/
9381	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9382	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9383
9384	uint32_t u32GuestCr4;
9385	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9386	AssertRCBreak(rc);
9387	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9388	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9389
9390	/*
9391	* IA32_DEBUGCTL MSR.
9392	*/
9393	uint64_t u64Val;
9394	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9395	AssertRCBreak(rc);
9396	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9397	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9398	{
9399	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9400	}
9401	uint64_t u64DebugCtlMsr = u64Val;
9402
9403	#ifdef VBOX_STRICT
9404	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9405	AssertRCBreak(rc);
9406	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9407	#endif
9408	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9409
9410	/*
9411	* RIP and RFLAGS.
9412	*/
9413	uint32_t u32Eflags;
9414	#if HC_ARCH_BITS == 64
9415	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9416	AssertRCBreak(rc);
9417	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9418	if ( !fLongModeGuest
9419	\|\| !pCtx->cs.Attr.n.u1Long)
9420	{
9421	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9422	}
9423	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9424	* must be identical if the "IA-32e mode guest" VM-entry
9425	* control is 1 and CS.L is 1. No check applies if the
9426	* CPU supports 64 linear-address bits. */
9427
9428	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9429	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9430	AssertRCBreak(rc);
9431	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9432	VMX_IGS_RFLAGS_RESERVED);
9433	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9434	u32Eflags = u64Val;
9435	#else
9436	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9437	AssertRCBreak(rc);
9438	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9439	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9440	#endif
9441
9442	if ( fLongModeGuest
9443	\|\| ( fUnrestrictedGuest
9444	&& !(u32GuestCr0 & X86_CR0_PE)))
9445	{
9446	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9447	}
9448
9449	uint32_t u32EntryInfo;
9450	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9451	AssertRCBreak(rc);
9452	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9453	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9454	{
9455	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9456	}
9457
9458	/*
9459	* 64-bit checks.
9460	*/
9461	#if HC_ARCH_BITS == 64
9462	if (fLongModeGuest)
9463	{
9464	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9465	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9466	}
9467
9468	if ( !fLongModeGuest
9469	&& (u32GuestCr4 & X86_CR4_PCIDE))
9470	{
9471	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9472	}
9473
9474	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9475	* 51:32 beyond the processor's physical-address width are 0. */
9476
9477	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9478	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9479	{
9480	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9481	}
9482
9483	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9484	AssertRCBreak(rc);
9485	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9486
9487	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9488	AssertRCBreak(rc);
9489	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9490	#endif
9491
9492	/*
9493	* PERF_GLOBAL MSR.
9494	*/
9495	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9496	{
9497	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9498	AssertRCBreak(rc);
9499	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9500	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9501	}
9502
9503	/*
9504	* PAT MSR.
9505	*/
9506	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9507	{
9508	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9509	AssertRCBreak(rc);
9510	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9511	for (unsigned i = 0; i < 8; i++)
9512	{
9513	uint8_t u8Val = (u64Val & 0xff);
9514	if ( u8Val != 0 /* UC */
9515	&& u8Val != 1 /* WC */
9516	&& u8Val != 4 /* WT */
9517	&& u8Val != 5 /* WP */
9518	&& u8Val != 6 /* WB */
9519	&& u8Val != 7 /* UC- */)
9520	{
9521	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9522	}
9523	u64Val >>= 8;
9524	}
9525	}
9526
9527	/*
9528	* EFER MSR.
9529	*/
9530	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9531	{
9532	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9533	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9534	AssertRCBreak(rc);
9535	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9536	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9537	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9538	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9539	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9540	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9541	* iemVmxVmentryCheckGuestState(). */
9542	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9543	\|\| !(u32GuestCr0 & X86_CR0_PG)
9544	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9545	VMX_IGS_EFER_LMA_LME_MISMATCH);
9546	}
9547
9548	/*
9549	* Segment registers.
9550	*/
9551	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9552	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9553	if (!(u32Eflags & X86_EFL_VM))
9554	{
9555	/* CS */
9556	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9557	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9558	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9559	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9560	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9561	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9562	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9563	/* CS cannot be loaded with NULL in protected mode. */
9564	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9565	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9566	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9567	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9568	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9569	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9570	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9571	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9572	else
9573	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9574
9575	/* SS */
9576	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9577	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9578	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9579	if ( !(pCtx->cr0 & X86_CR0_PE)
9580	\|\| pCtx->cs.Attr.n.u4Type == 3)
9581	{
9582	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9583	}
9584	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9585	{
9586	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9587	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9588	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9589	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9590	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9591	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9592	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9593	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9594	}
9595
9596	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9597	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9598	{
9599	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9600	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9601	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9602	\|\| pCtx->ds.Attr.n.u4Type > 11
9603	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9604	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9605	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9606	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9607	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9608	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9609	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9610	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9611	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9612	}
9613	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9614	{
9615	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9616	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9617	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9618	\|\| pCtx->es.Attr.n.u4Type > 11
9619	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9620	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9621	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9622	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9623	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9624	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9625	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9626	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9627	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9628	}
9629	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9630	{
9631	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9632	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9633	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9634	\|\| pCtx->fs.Attr.n.u4Type > 11
9635	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9636	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9637	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9638	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9639	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9640	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9641	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9642	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9643	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9644	}
9645	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9646	{
9647	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9648	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9649	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9650	\|\| pCtx->gs.Attr.n.u4Type > 11
9651	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9652	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9653	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9654	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9655	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9656	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9657	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9658	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9659	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9660	}
9661	/* 64-bit capable CPUs. */
9662	#if HC_ARCH_BITS == 64
9663	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9664	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9665	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9666	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9667	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9668	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9669	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9670	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9671	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9672	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9673	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9674	#endif
9675	}
9676	else
9677	{
9678	/* V86 mode checks. */
9679	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9680	if (pVmcsInfo->RealMode.fRealOnV86Active)
9681	{
9682	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9683	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9684	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9685	}
9686	else
9687	{
9688	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9689	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9690	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9691	}
9692
9693	/* CS */
9694	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9695	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9696	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9697	/* SS */
9698	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9699	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9700	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9701	/* DS */
9702	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9703	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9704	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9705	/* ES */
9706	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9707	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9708	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9709	/* FS */
9710	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9711	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9712	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9713	/* GS */
9714	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9715	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9716	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9717	/* 64-bit capable CPUs. */
9718	#if HC_ARCH_BITS == 64
9719	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9720	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9721	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9722	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9723	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9724	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9725	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9726	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9727	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9728	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9729	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9730	#endif
9731	}
9732
9733	/*
9734	* TR.
9735	*/
9736	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9737	/* 64-bit capable CPUs. */
9738	#if HC_ARCH_BITS == 64
9739	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9740	#endif
9741	if (fLongModeGuest)
9742	{
9743	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9744	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9745	}
9746	else
9747	{
9748	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9749	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9750	VMX_IGS_TR_ATTR_TYPE_INVALID);
9751	}
9752	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9753	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9754	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9755	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9756	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9757	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9758	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9759	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9760
9761	/*
9762	* GDTR and IDTR.
9763	*/
9764	#if HC_ARCH_BITS == 64
9765	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9766	AssertRCBreak(rc);
9767	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9768
9769	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9770	AssertRCBreak(rc);
9771	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9772	#endif
9773
9774	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9775	AssertRCBreak(rc);
9776	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9777
9778	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9779	AssertRCBreak(rc);
9780	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9781
9782	/*
9783	* Guest Non-Register State.
9784	*/
9785	/* Activity State. */
9786	uint32_t u32ActivityState;
9787	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9788	AssertRCBreak(rc);
9789	HMVMX_CHECK_BREAK( !u32ActivityState
9790	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9791	VMX_IGS_ACTIVITY_STATE_INVALID);
9792	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9793	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9794	uint32_t u32IntrState;
9795	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9796	AssertRCBreak(rc);
9797	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9798	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9799	{
9800	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9801	}
9802
9803	/** @todo Activity state and injecting interrupts. Left as a todo since we
9804	* currently don't use activity states but ACTIVE. */
9805
9806	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9807	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9808
9809	/* Guest interruptibility-state. */
9810	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9811	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9812	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9813	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9814	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9815	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9816	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9817	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9818	{
9819	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9820	{
9821	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9822	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9823	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9824	}
9825	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9826	{
9827	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9828	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9829	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9830	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9831	}
9832	}
9833	/** @todo Assumes the processor is not in SMM. */
9834	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9835	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9836	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9837	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9838	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9839	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9840	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9841	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9842	{
9843	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9844	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9845	}
9846
9847	/* Pending debug exceptions. */
9848	#if HC_ARCH_BITS == 64
9849	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9850	AssertRCBreak(rc);
9851	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9852	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9853	u32Val = u64Val; /* For pending debug exceptions checks below. */
9854	#else
9855	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9856	AssertRCBreak(rc);
9857	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9858	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9859	#endif
9860
9861	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9862	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9863	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9864	{
9865	if ( (u32Eflags & X86_EFL_TF)
9866	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9867	{
9868	/* Bit 14 is PendingDebug.BS. */
9869	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9870	}
9871	if ( !(u32Eflags & X86_EFL_TF)
9872	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9873	{
9874	/* Bit 14 is PendingDebug.BS. */
9875	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9876	}
9877	}
9878
9879	/* VMCS link pointer. */
9880	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9881	AssertRCBreak(rc);
9882	if (u64Val != UINT64_C(0xffffffffffffffff))
9883	{
9884	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9885	/** @todo Bits beyond the processor's physical-address width MBZ. */
9886	/** @todo 32-bit located in memory referenced by value of this field (as a
9887	* physical address) must contain the processor's VMCS revision ID. */
9888	/** @todo SMM checks. */
9889	}
9890
9891	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9892	* not using nested paging? */
9893	if ( pVM->hm.s.fNestedPaging
9894	&& !fLongModeGuest
9895	&& CPUMIsGuestInPAEModeEx(pCtx))
9896	{
9897	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9898	AssertRCBreak(rc);
9899	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9900
9901	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9902	AssertRCBreak(rc);
9903	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9904
9905	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9906	AssertRCBreak(rc);
9907	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9908
9909	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9910	AssertRCBreak(rc);
9911	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9912	}
9913
9914	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9915	if (uError == VMX_IGS_ERROR)
9916	uError = VMX_IGS_REASON_NOT_FOUND;
9917	} while (0);
9918
9919	pVCpu->hm.s.u32HMError = uError;
9920	return uError;
9921
9922	#undef HMVMX_ERROR_BREAK
9923	#undef HMVMX_CHECK_BREAK
9924	}
9925
9926
9927	/**
9928	* Setup the APIC-access page for virtualizing APIC access.
9929	*
9930	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9931	* this not done as part of exporting guest state, see @bugref{8721}.
9932	*
9933	* @returns VBox status code.
9934	* @param pVCpu The cross context virtual CPU structure.
9935	*/
9936	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9937	{
9938	PVM pVM = pVCpu->CTX_SUFF(pVM);
9939	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9940
9941	Assert(PDMHasApic(pVM));
9942	Assert(u64MsrApicBase);
9943
9944	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9945	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9946
9947	/* Unalias any existing mapping. */
9948	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9949	AssertRCReturn(rc, rc);
9950
9951	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9952	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9953	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9954	AssertRCReturn(rc, rc);
9955
9956	/* Update the per-VCPU cache of the APIC base MSR. */
9957	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9958	return VINF_SUCCESS;
9959	}
9960
9961
9962	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9963	/**
9964	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9965	* nested-guest using hardware-assisted VMX.
9966	*
9967	* @param pVCpu The cross context virtual CPU structure.
9968	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9969	* @param pVmcsInfoGst The guest VMCS info. object.
9970	*/
9971	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9972	{
9973	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9974	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9975	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9976	Assert(pu64MsrBitmapNstGst);
9977	Assert(pu64MsrBitmapGst);
9978	Assert(pu64MsrBitmap);
9979
9980	/*
9981	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9982	* MSR that is intercepted by the guest is also intercepted while executing the
9983	* nested-guest using hardware-assisted VMX.
9984	*/
9985	uint32_t const cbFrag = sizeof(uint64_t);
9986	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9987	for (uint32_t i = 0; i <= cFrags; i++)
9988	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9989	}
9990
9991
9992	/**
9993	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9994	* hardware-assisted VMX execution of the nested-guest.
9995	*
9996	* For a guest, we don't modify these controls once we set up the VMCS and hence
9997	* this function is never called.
9998	*
9999	* For nested-guests since the guest hypervisor provides these controls on every
10000	* nested-guest VM-entry and could potentially change them everytime we need to
10001	* merge them before every nested-guest VM-entry.
10002	*
10003	* @returns VBox status code.
10004	* @param pVCpu The cross context virtual CPU structure.
10005	*/
10006	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10007	{
10008	PVM pVM = pVCpu->CTX_SUFF(pVM);
10009	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10010	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10011	Assert(pVmcsNstGst);
10012
10013	/*
10014	* Merge the controls with the requirements of the guest VMCS.
10015	*
10016	* We do not need to validate the nested-guest VMX features specified in the
10017	* nested-guest VMCS with the features supported by the physical CPU as it's
10018	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10019	*
10020	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10021	* guest are derived from the VMX features supported by the physical CPU.
10022	*/
10023
10024	/* Pin-based VM-execution controls. */
10025	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10026
10027	/* Processor-based VM-execution controls. */
10028	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10029	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10030	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10031	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10032	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10033
10034	/* Secondary processor-based VM-execution controls. */
10035	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10036	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10037	\| VMX_PROC_CTLS2_INVPCID
10038	\| VMX_PROC_CTLS2_RDTSCP
10039	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10040	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10041	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10042	\| VMX_PROC_CTLS2_VMFUNC));
10043
10044	/*
10045	* VM-entry controls:
10046	* These controls contains state that depends on the nested-guest state (primarily
10047	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10048	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10049	* properly continue executing the nested-guest if the EFER MSR changes but does not
10050	* cause a nested-guest VM-exits.
10051	*
10052	* VM-exit controls:
10053	* These controls specify the host state on return. We cannot use the controls from
10054	* the nested-hypervisor state as is as it would contain the guest state rather than
10055	* the host state. Since the host state is subject to change (e.g. preemption, trips
10056	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10057	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10058	*
10059	* VM-entry MSR-load:
10060	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10061	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10062	*
10063	* VM-exit MSR-store:
10064	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10065	* context back into the VM-exit MSR-store area.
10066	*
10067	* VM-exit MSR-load areas:
10068	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10069	* we can entirely ignore what the nested-hypervisor wants to load here.
10070	*/
10071
10072	/*
10073	* Exception bitmap.
10074	*
10075	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10076	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10077	* keep the code more flexible if intercepting exceptions become more dynamic in
10078	* the future we do it as part of exporting the nested-guest state.
10079	*/
10080	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10081
10082	/*
10083	* CR0/CR4 guest/host mask.
10084	*
10085	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10086	* must cause VM-exits, so we need to merge them here.
10087	*/
10088	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10089	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10090
10091	/*
10092	* Page-fault error-code mask and match.
10093	*
10094	* Although we require unrestricted guest execution (and thereby nested-paging) for
10095	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10096	* normally intercept #PFs, it might intercept them for debugging purposes.
10097	*
10098	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10099	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10100	*/
10101	uint32_t u32XcptPFMask;
10102	uint32_t u32XcptPFMatch;
10103	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10104	{
10105	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10106	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10107	}
10108	else
10109	{
10110	u32XcptPFMask = 0;
10111	u32XcptPFMatch = 0;
10112	}
10113
10114	/*
10115	* Pause-Loop exiting.
10116	*/
10117	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10118	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10119
10120	/*
10121	* I/O Bitmap.
10122	*
10123	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10124	* always intercept all I/O port accesses.
10125	*/
10126	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10127
10128	/*
10129	* APIC-access page.
10130	*
10131	* The APIC-access page address has already been initialized while setting up the
10132	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10133	* should not be on any consequence to the host or to the guest for that matter, but
10134	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10135	* emulation to keep it simple.
10136	*/
10137
10138	/*
10139	* Virtual-APIC page and TPR threshold.
10140	*
10141	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10142	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10143	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10144	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10145	*/
10146	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10147	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10148	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10149	{
10150	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10151
10152	/*
10153	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10154	* we would fail to obtain a valid host-physical address for its guest-physical
10155	* address.
10156	*
10157	* We currently do not support this scenario. Maybe in the future if there is a
10158	* pressing need we can explore making this particular set of conditions work.
10159	* Right now we just cause a VM-entry failure.
10160	*
10161	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10162	* so should not really failure at the moment.
10163	*/
10164	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10165	}
10166	else
10167	{
10168	/*
10169	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10170	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10171	* be taken care of by EPT/shadow paging.
10172	*/
10173	if (pVM->hm.s.fAllow64BitGuests)
10174	{
10175	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10176	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10177	}
10178	}
10179
10180	/*
10181	* Validate basic assumptions.
10182	*/
10183	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10184	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10185	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10186	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10187
10188	/*
10189	* Commit it to the nested-guest VMCS.
10190	*/
10191	int rc = VINF_SUCCESS;
10192	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10193	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10194	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10195	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10196	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10197	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10198	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10199	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10200	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10201	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10202	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10203	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10204	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10205	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10206	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10207	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10208	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10209	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10210	{
10211	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10212	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10213	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10214	}
10215	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10216	{
10217	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10218	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10219	}
10220	AssertRCReturn(rc, rc);
10221
10222	/*
10223	* Update the nested-guest VMCS cache.
10224	*/
10225	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10226	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10227	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10228	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10229	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10230	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10231	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10232	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10233	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10234
10235	/*
10236	* MSR bitmap.
10237	*
10238	* The MSR bitmap address has already been initialized while setting up the
10239	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10240	*/
10241	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10242	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10243
10244	return VINF_SUCCESS;
10245	}
10246	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10247
10248
10249	/**
10250	* Does the preparations before executing guest code in VT-x.
10251	*
10252	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10253	* recompiler/IEM. We must be cautious what we do here regarding committing
10254	* guest-state information into the VMCS assuming we assuredly execute the
10255	* guest in VT-x mode.
10256	*
10257	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10258	* the common-state (TRPM/forceflags), we must undo those changes so that the
10259	* recompiler/IEM can (and should) use them when it resumes guest execution.
10260	* Otherwise such operations must be done when we can no longer exit to ring-3.
10261	*
10262	* @returns Strict VBox status code (i.e. informational status codes too).
10263	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10264	* have been disabled.
10265	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10266	* double-fault into the guest.
10267	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10268	* dispatched directly.
10269	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10270	*
10271	* @param pVCpu The cross context virtual CPU structure.
10272	* @param pVmxTransient The VMX-transient structure.
10273	* @param fStepping Whether we are single-stepping the guest in the
10274	* hypervisor debugger. Makes us ignore some of the reasons
10275	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10276	* if event dispatching took place.
10277	*/
10278	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10279	{
10280	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10281
10282	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10283	if (pVmxTransient->fIsNestedGuest)
10284	{
10285	RT_NOREF2(pVCpu, fStepping);
10286	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10287	return VINF_EM_RESCHEDULE_REM;
10288	}
10289	#endif
10290
10291	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10292	PGMRZDynMapFlushAutoSet(pVCpu);
10293	#endif
10294
10295	/*
10296	* Check and process force flag actions, some of which might require us to go back to ring-3.
10297	*/
10298	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10299	if (rcStrict == VINF_SUCCESS)
10300	{ /* FFs don't get set all the time. */ }
10301	else
10302	return rcStrict;
10303
10304	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10305	/*
10306	* Switch to the nested-guest VMCS as we may have transitioned into executing
10307	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10308	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10309	*
10310	* We do this as late as possible to minimize (though not completely remove)
10311	* clearing/loading VMCS again due to premature trips to ring-3 above.
10312	*/
10313	if (pVmxTransient->fIsNestedGuest)
10314	{
10315	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10316	{
10317	/*
10318	* Ensure we have synced everything from the guest VMCS and also flag that
10319	* that we need to export the full (nested) guest-CPU context to the
10320	* nested-guest VMCS.
10321	*/
10322	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10323	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10324
10325	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10326	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10327	if (RT_LIKELY(rc == VINF_SUCCESS))
10328	{
10329	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10330	ASMSetFlags(fEFlags);
10331	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10332
10333	/*
10334	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10335	* flag that we need to update the host MSR values there. Even if we decide
10336	* in the future to share the VM-exit MSR-store area page with the guest,
10337	* if its content differs, we would have to update the host MSRs anyway.
10338	*/
10339	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10340	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10341	}
10342	else
10343	{
10344	ASMSetFlags(fEFlags);
10345	return rc;
10346	}
10347	}
10348
10349	/*
10350	* Merge guest VMCS controls with the nested-guest VMCS controls.
10351	*
10352	* Even if we have not executed the guest prior to this (e.g. when resuming
10353	* from a saved state), we should be okay with merging controls as we
10354	* initialize the guest VMCS controls as part of VM setup phase.
10355	*/
10356	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10357	{
10358	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10359	AssertRCReturn(rc, rc);
10360	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10361	}
10362	}
10363	#endif
10364
10365	/*
10366	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10367	* We look at the guest VMCS control here as we always set it when supported by
10368	* the physical CPU. Looking at the nested-guest control here would not be
10369	* possible because they are not merged yet.
10370	*/
10371	PVM pVM = pVCpu->CTX_SUFF(pVM);
10372	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10373	Assert(pVmcsInfo);
10374	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10375	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10376	&& PDMHasApic(pVM))
10377	{
10378	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10379	AssertRCReturn(rc, rc);
10380	}
10381
10382	/*
10383	* Evaluate events to be injected into the guest.
10384	*
10385	* Events in TRPM can be injected without inspecting the guest state.
10386	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10387	* guest to cause a VM-exit the next time they are ready to receive the event.
10388	*/
10389	if (TRPMHasTrap(pVCpu))
10390	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10391
10392	uint32_t fIntrState;
10393	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10394
10395	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10396	/*
10397	* While evaluating pending events if something failed (unlikely) or if we were
10398	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10399	*/
10400	if ( rcStrict != VINF_SUCCESS
10401	\|\| ( pVmxTransient->fIsNestedGuest
10402	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10403	return rcStrict;
10404	#endif
10405
10406	/*
10407	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10408	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10409	* also result in triple-faulting the VM.
10410	*
10411	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10412	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10413	*/
10414	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10415	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10416	{ /* likely */ }
10417	else
10418	{
10419	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10420	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10421	return rcStrict;
10422	}
10423
10424	/*
10425	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10426	* import CR3 themselves. We will need to update them here, as even as late as the above
10427	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10428	* the below force flags to be set.
10429	*/
10430	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10431	{
10432	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10433	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10434	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10435	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10436	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10437	}
10438	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10439	{
10440	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10441	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10442	}
10443
10444	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10445	/* Paranoia. */
10446	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10447	#endif
10448
10449	/*
10450	* No longjmps to ring-3 from this point on!!!
10451	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10452	* This also disables flushing of the R0-logger instance (if any).
10453	*/
10454	VMMRZCallRing3Disable(pVCpu);
10455
10456	/*
10457	* Export the guest state bits.
10458	*
10459	* We cannot perform longjmps while loading the guest state because we do not preserve the
10460	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10461	* CPU migration.
10462	*
10463	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10464	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10465	*/
10466	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10467	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10468	{ /* likely */ }
10469	else
10470	{
10471	VMMRZCallRing3Enable(pVCpu);
10472	return rcStrict;
10473	}
10474
10475	/*
10476	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10477	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10478	* preemption disabled for a while. Since this is purely to aid the
10479	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10480	* disable interrupt on NT.
10481	*
10482	* We need to check for force-flags that could've possible been altered since we last
10483	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10484	* see @bugref{6398}).
10485	*
10486	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10487	* to ring-3 before executing guest code.
10488	*/
10489	pVmxTransient->fEFlags = ASMIntDisableFlags();
10490
10491	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10492	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10493	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10494	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10495	{
10496	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10497	{
10498	pVCpu->hm.s.Event.fPending = false;
10499
10500	/*
10501	* We've injected any pending events. This is really the point of no return (to ring-3).
10502	*
10503	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10504	* returns from this function, so don't enable them here.
10505	*/
10506	return VINF_SUCCESS;
10507	}
10508
10509	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10510	rcStrict = VINF_EM_RAW_INTERRUPT;
10511	}
10512	else
10513	{
10514	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10515	rcStrict = VINF_EM_RAW_TO_R3;
10516	}
10517
10518	ASMSetFlags(pVmxTransient->fEFlags);
10519	VMMRZCallRing3Enable(pVCpu);
10520
10521	return rcStrict;
10522	}
10523
10524
10525	/**
10526	* Final preparations before executing guest code using hardware-assisted VMX.
10527	*
10528	* We can no longer get preempted to a different host CPU and there are no returns
10529	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10530	* failures), this function is not intended to fail sans unrecoverable hardware
10531	* errors.
10532	*
10533	* @param pVCpu The cross context virtual CPU structure.
10534	* @param pVmxTransient The VMX-transient structure.
10535	*
10536	* @remarks Called with preemption disabled.
10537	* @remarks No-long-jump zone!!!
10538	*/
10539	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10540	{
10541	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10542	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10543	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10544	Assert(!pVCpu->hm.s.Event.fPending);
10545
10546	/*
10547	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10548	*/
10549	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10550	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10551
10552	PVM pVM = pVCpu->CTX_SUFF(pVM);
10553	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10554
10555	if (!CPUMIsGuestFPUStateActive(pVCpu))
10556	{
10557	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10558	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10559	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10560	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10561	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10562	}
10563
10564	/*
10565	* Re-save the host state bits as we may've been preempted (only happens when
10566	* thread-context hooks are used or when the VM start function changes).
10567	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10568	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10569	* see @bugref{8432}.
10570	*
10571	* This may also happen when switching to/from a nested-guest VMCS without leaving
10572	* ring-0.
10573	*/
10574	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10575	{
10576	int rc = hmR0VmxExportHostState(pVCpu);
10577	AssertRC(rc);
10578	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10579	}
10580	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10581
10582	/*
10583	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10584	*/
10585	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10586	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10587	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10588
10589	/*
10590	* Store status of the shared guest/host debug state at the time of VM-entry.
10591	*/
10592	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10593	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10594	{
10595	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10596	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10597	}
10598	else
10599	#endif
10600	{
10601	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10602	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10603	}
10604
10605	/*
10606	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10607	* more than one conditional check. The post-run side of our code shall determine
10608	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10609	*/
10610	if (pVmcsInfo->pbVirtApic)
10611	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10612
10613	/*
10614	* Update the host MSRs values in the VM-exit MSR-load area.
10615	*/
10616	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10617	{
10618	if (pVmcsInfo->cExitMsrLoad > 0)
10619	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10620	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10621	}
10622
10623	/*
10624	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10625	* VMX-preemption timer based on the next virtual sync clock deadline.
10626	*/
10627	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10628	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10629	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10630	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10631	{
10632	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10633	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10634	}
10635
10636	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10637	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10638	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10639	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10640
10641	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10642
10643	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10644	as we're about to start executing the guest . */
10645
10646	/*
10647	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10648	*
10649	* This is done this late as updating the TSC offsetting/preemption timer above
10650	* figures out if we can skip intercepting RDTSCP by calculating the number of
10651	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10652	*/
10653	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10654	{
10655	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10656	{
10657	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10658	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10659	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10660	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10661	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10662	AssertRC(rc);
10663	}
10664	else
10665	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10666	}
10667
10668	#ifdef VBOX_STRICT
10669	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10670	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10671	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10672	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10673	#endif
10674
10675	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10676	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10677	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10678	* see @bugref{9180#c54}. */
10679	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10680	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10681	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10682	#endif
10683	}
10684
10685
10686	/**
10687	* First C routine invoked after running guest code using hardware-assisted VMX.
10688	*
10689	* @param pVCpu The cross context virtual CPU structure.
10690	* @param pVmxTransient The VMX-transient structure.
10691	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10692	*
10693	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10694	*
10695	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10696	* unconditionally when it is safe to do so.
10697	*/
10698	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10699	{
10700	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10701
10702	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10703	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10704	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10705	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10706	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10707	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10708
10709	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10710	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10711	{
10712	uint64_t uGstTsc;
10713	if (!pVmxTransient->fIsNestedGuest)
10714	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10715	else
10716	{
10717	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10718	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10719	}
10720	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10721	}
10722
10723	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10724	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10725	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10726
10727	#if HC_ARCH_BITS == 64
10728	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10729	#endif
10730	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10731	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10732	and we need to leave it alone here. */
10733	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10734	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10735	#else
10736	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10737	#endif
10738	#ifdef VBOX_STRICT
10739	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10740	#endif
10741	Assert(!ASMIntAreEnabled());
10742	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10743	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10744
10745	/*
10746	* Save the basic VM-exit reason and check if the VM-entry failed.
10747	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10748	*/
10749	uint32_t uExitReason;
10750	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10751	AssertRC(rc);
10752	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10753	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10754
10755	/*
10756	* Check if VMLAUNCH/VMRESUME succeeded.
10757	* If this failed, we cause a guru meditation and cease further execution.
10758	*
10759	* However, if we are executing a nested-guest we might fail if we use the
10760	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10761	*/
10762	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10763	{
10764	/*
10765	* Update the VM-exit history array here even if the VM-entry failed due to:
10766	* - Invalid guest state.
10767	* - MSR loading.
10768	* - Machine-check event.
10769	*
10770	* In any of the above cases we will still have a "valid" VM-exit reason
10771	* despite @a fVMEntryFailed being false.
10772	*
10773	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10774	*
10775	* Note! We don't have CS or RIP at this point. Will probably address that later
10776	* by amending the history entry added here.
10777	*/
10778	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10779	UINT64_MAX, uHostTsc);
10780
10781	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10782	{
10783	VMMRZCallRing3Enable(pVCpu);
10784
10785	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10786	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10787
10788	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10789	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10790	AssertRC(rc);
10791	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10792	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10793	AssertRC(rc);
10794	#else
10795	/*
10796	* Import the guest-interruptibility state always as we need it while evaluating
10797	* injecting events on re-entry.
10798	*
10799	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10800	* checking for real-mode while exporting the state because all bits that cause
10801	* mode changes wrt CR0 are intercepted.
10802	*/
10803	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10804	AssertRC(rc);
10805	#endif
10806
10807	/*
10808	* Sync the TPR shadow with our APIC state.
10809	*/
10810	if ( !pVmxTransient->fIsNestedGuest
10811	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10812	{
10813	Assert(pVmcsInfo->pbVirtApic);
10814	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10815	{
10816	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10817	AssertRC(rc);
10818	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10819	}
10820	}
10821
10822	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10823	return;
10824	}
10825	}
10826	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10827	else if (pVmxTransient->fIsNestedGuest)
10828	{
10829	# if 0
10830	/*
10831	* Copy the VM-instruction error field to the guest VMCS.
10832	*/
10833	/** @todo NSTVMX: Verify we're using the fast path. */
10834	uint32_t u32RoVmInstrError;
10835	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10836	AssertRCReturn(rc, rc);
10837	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10838	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10839	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10840	# else
10841	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10842	# endif
10843	}
10844	#endif
10845	else
10846	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10847
10848	VMMRZCallRing3Enable(pVCpu);
10849	}
10850
10851
10852	/**
10853	* Runs the guest code using hardware-assisted VMX the normal way.
10854	*
10855	* @returns VBox status code.
10856	* @param pVCpu The cross context virtual CPU structure.
10857	* @param pcLoops Pointer to the number of executed loops.
10858	*/
10859	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10860	{
10861	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10862	Assert(pcLoops);
10863	Assert(*pcLoops <= cMaxResumeLoops);
10864
10865	VMXTRANSIENT VmxTransient;
10866	RT_ZERO(VmxTransient);
10867	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10868
10869	/* Paranoia. */
10870	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10871	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10872
10873	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10874	for (;;)
10875	{
10876	Assert(!HMR0SuspendPending());
10877	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10878	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10879
10880	/*
10881	* Preparatory work for running nested-guest code, this may force us to
10882	* return to ring-3.
10883	*
10884	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10885	*/
10886	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10887	if (rcStrict != VINF_SUCCESS)
10888	break;
10889
10890	/* Interrupts are disabled at this point! */
10891	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10892	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10893	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10894	/* Interrupts are re-enabled at this point! */
10895
10896	/*
10897	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10898	*/
10899	if (RT_SUCCESS(rcRun))
10900	{ /* very likely */ }
10901	else
10902	{
10903	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10904	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10905	return rcRun;
10906	}
10907
10908	/*
10909	* Profile the VM-exit.
10910	*/
10911	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10913	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10914	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10915	HMVMX_START_EXIT_DISPATCH_PROF();
10916
10917	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10918
10919	/*
10920	* Handle the VM-exit.
10921	*/
10922	#ifdef HMVMX_USE_FUNCTION_TABLE
10923	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10924	#else
10925	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10926	#endif
10927	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10928	if (rcStrict == VINF_SUCCESS)
10929	{
10930	if (++(*pcLoops) <= cMaxResumeLoops)
10931	continue;
10932	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10933	rcStrict = VINF_EM_RAW_INTERRUPT;
10934	}
10935	break;
10936	}
10937
10938	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10939	return rcStrict;
10940	}
10941
10942	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10943	/**
10944	* Runs the nested-guest code using hardware-assisted VMX.
10945	*
10946	* @returns VBox status code.
10947	* @param pVCpu The cross context virtual CPU structure.
10948	* @param pcLoops Pointer to the number of executed loops.
10949	*
10950	* @sa hmR0VmxRunGuestCodeNormal.
10951	*/
10952	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10953	{
10954	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10955	Assert(pcLoops);
10956	Assert(*pcLoops <= cMaxResumeLoops);
10957
10958	VMXTRANSIENT VmxTransient;
10959	RT_ZERO(VmxTransient);
10960	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10961	VmxTransient.fIsNestedGuest = true;
10962
10963	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10964	for (;;)
10965	{
10966	Assert(!HMR0SuspendPending());
10967	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10968	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10969
10970	/*
10971	* Preparatory work for running guest code, this may force us to
10972	* return to ring-3.
10973	*
10974	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10975	*/
10976	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10977	if (rcStrict != VINF_SUCCESS)
10978	break;
10979
10980	/* Interrupts are disabled at this point! */
10981	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10982	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10983	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10984	/* Interrupts are re-enabled at this point! */
10985
10986	/*
10987	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10988	*/
10989	if (RT_SUCCESS(rcRun))
10990	{ /* very likely */ }
10991	else
10992	{
10993	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10994	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10995	return rcRun;
10996	}
10997
10998	/*
10999	* Profile the VM-exit.
11000	*/
11001	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11003	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11004	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11005	HMVMX_START_EXIT_DISPATCH_PROF();
11006
11007	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11008
11009	/*
11010	* Handle the VM-exit.
11011	*/
11012	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11013	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11014	if ( rcStrict == VINF_SUCCESS
11015	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11016	{
11017	if (++(*pcLoops) <= cMaxResumeLoops)
11018	continue;
11019	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11020	rcStrict = VINF_EM_RAW_INTERRUPT;
11021	}
11022	break;
11023	}
11024
11025	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11026	return rcStrict;
11027	}
11028	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11029
11030
11031	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11032	* probes.
11033	*
11034	* The following few functions and associated structure contains the bloat
11035	* necessary for providing detailed debug events and dtrace probes as well as
11036	* reliable host side single stepping. This works on the principle of
11037	* "subclassing" the normal execution loop and workers. We replace the loop
11038	* method completely and override selected helpers to add necessary adjustments
11039	* to their core operation.
11040	*
11041	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11042	* any performance for debug and analysis features.
11043	*
11044	* @{
11045	*/
11046
11047	/**
11048	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11049	* the debug run loop.
11050	*/
11051	typedef struct VMXRUNDBGSTATE
11052	{
11053	/** The RIP we started executing at. This is for detecting that we stepped. */
11054	uint64_t uRipStart;
11055	/** The CS we started executing with. */
11056	uint16_t uCsStart;
11057
11058	/** Whether we've actually modified the 1st execution control field. */
11059	bool fModifiedProcCtls : 1;
11060	/** Whether we've actually modified the 2nd execution control field. */
11061	bool fModifiedProcCtls2 : 1;
11062	/** Whether we've actually modified the exception bitmap. */
11063	bool fModifiedXcptBitmap : 1;
11064
11065	/** We desire the modified the CR0 mask to be cleared. */
11066	bool fClearCr0Mask : 1;
11067	/** We desire the modified the CR4 mask to be cleared. */
11068	bool fClearCr4Mask : 1;
11069	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11070	uint32_t fCpe1Extra;
11071	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11072	uint32_t fCpe1Unwanted;
11073	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11074	uint32_t fCpe2Extra;
11075	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11076	uint32_t bmXcptExtra;
11077	/** The sequence number of the Dtrace provider settings the state was
11078	* configured against. */
11079	uint32_t uDtraceSettingsSeqNo;
11080	/** VM-exits to check (one bit per VM-exit). */
11081	uint32_t bmExitsToCheck[3];
11082
11083	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11084	uint32_t fProcCtlsInitial;
11085	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11086	uint32_t fProcCtls2Initial;
11087	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11088	uint32_t bmXcptInitial;
11089	} VMXRUNDBGSTATE;
11090	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11091	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11092
11093
11094	/**
11095	* Initializes the VMXRUNDBGSTATE structure.
11096	*
11097	* @param pVCpu The cross context virtual CPU structure of the
11098	* calling EMT.
11099	* @param pVmxTransient The VMX-transient structure.
11100	* @param pDbgState The debug state to initialize.
11101	*/
11102	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11103	{
11104	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11105	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11106
11107	pDbgState->fModifiedProcCtls = false;
11108	pDbgState->fModifiedProcCtls2 = false;
11109	pDbgState->fModifiedXcptBitmap = false;
11110	pDbgState->fClearCr0Mask = false;
11111	pDbgState->fClearCr4Mask = false;
11112	pDbgState->fCpe1Extra = 0;
11113	pDbgState->fCpe1Unwanted = 0;
11114	pDbgState->fCpe2Extra = 0;
11115	pDbgState->bmXcptExtra = 0;
11116	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11117	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11118	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11119	}
11120
11121
11122	/**
11123	* Updates the VMSC fields with changes requested by @a pDbgState.
11124	*
11125	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11126	* immediately before executing guest code, i.e. when interrupts are disabled.
11127	* We don't check status codes here as we cannot easily assert or return in the
11128	* latter case.
11129	*
11130	* @param pVCpu The cross context virtual CPU structure.
11131	* @param pVmxTransient The VMX-transient structure.
11132	* @param pDbgState The debug state.
11133	*/
11134	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11135	{
11136	/*
11137	* Ensure desired flags in VMCS control fields are set.
11138	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11139	*
11140	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11141	* there should be no stale data in pCtx at this point.
11142	*/
11143	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11144	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11145	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11146	{
11147	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11148	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11149	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11150	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11151	pDbgState->fModifiedProcCtls = true;
11152	}
11153
11154	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11155	{
11156	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11157	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11158	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11159	pDbgState->fModifiedProcCtls2 = true;
11160	}
11161
11162	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11163	{
11164	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11165	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11166	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11167	pDbgState->fModifiedXcptBitmap = true;
11168	}
11169
11170	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11171	{
11172	pVmcsInfo->u64Cr0Mask = 0;
11173	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11174	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11175	}
11176
11177	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11178	{
11179	pVmcsInfo->u64Cr4Mask = 0;
11180	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11181	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11182	}
11183
11184	NOREF(pVCpu);
11185	}
11186
11187
11188	/**
11189	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11190	* re-entry next time around.
11191	*
11192	* @returns Strict VBox status code (i.e. informational status codes too).
11193	* @param pVCpu The cross context virtual CPU structure.
11194	* @param pVmxTransient The VMX-transient structure.
11195	* @param pDbgState The debug state.
11196	* @param rcStrict The return code from executing the guest using single
11197	* stepping.
11198	*/
11199	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11200	VBOXSTRICTRC rcStrict)
11201	{
11202	/*
11203	* Restore VM-exit control settings as we may not reenter this function the
11204	* next time around.
11205	*/
11206	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11207
11208	/* We reload the initial value, trigger what we can of recalculations the
11209	next time around. From the looks of things, that's all that's required atm. */
11210	if (pDbgState->fModifiedProcCtls)
11211	{
11212	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11213	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11214	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11215	AssertRCReturn(rc2, rc2);
11216	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11217	}
11218
11219	/* We're currently the only ones messing with this one, so just restore the
11220	cached value and reload the field. */
11221	if ( pDbgState->fModifiedProcCtls2
11222	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11223	{
11224	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11225	AssertRCReturn(rc2, rc2);
11226	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11227	}
11228
11229	/* If we've modified the exception bitmap, we restore it and trigger
11230	reloading and partial recalculation the next time around. */
11231	if (pDbgState->fModifiedXcptBitmap)
11232	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11233
11234	return rcStrict;
11235	}
11236
11237
11238	/**
11239	* Configures VM-exit controls for current DBGF and DTrace settings.
11240	*
11241	* This updates @a pDbgState and the VMCS execution control fields to reflect
11242	* the necessary VM-exits demanded by DBGF and DTrace.
11243	*
11244	* @param pVCpu The cross context virtual CPU structure.
11245	* @param pVmxTransient The VMX-transient structure. May update
11246	* fUpdatedTscOffsettingAndPreemptTimer.
11247	* @param pDbgState The debug state.
11248	*/
11249	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11250	{
11251	/*
11252	* Take down the dtrace serial number so we can spot changes.
11253	*/
11254	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11255	ASMCompilerBarrier();
11256
11257	/*
11258	* We'll rebuild most of the middle block of data members (holding the
11259	* current settings) as we go along here, so start by clearing it all.
11260	*/
11261	pDbgState->bmXcptExtra = 0;
11262	pDbgState->fCpe1Extra = 0;
11263	pDbgState->fCpe1Unwanted = 0;
11264	pDbgState->fCpe2Extra = 0;
11265	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11266	pDbgState->bmExitsToCheck[i] = 0;
11267
11268	/*
11269	* Software interrupts (INT XXh) - no idea how to trigger these...
11270	*/
11271	PVM pVM = pVCpu->CTX_SUFF(pVM);
11272	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11273	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11274	{
11275	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11276	}
11277
11278	/*
11279	* INT3 breakpoints - triggered by #BP exceptions.
11280	*/
11281	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11282	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11283
11284	/*
11285	* Exception bitmap and XCPT events+probes.
11286	*/
11287	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11288	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11289	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11290
11291	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11292	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11293	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11294	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11295	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11296	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11297	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11298	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11299	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11300	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11301	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11302	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11303	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11304	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11305	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11306	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11307	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11308	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11309
11310	if (pDbgState->bmXcptExtra)
11311	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11312
11313	/*
11314	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11315	*
11316	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11317	* So, when adding/changing/removing please don't forget to update it.
11318	*
11319	* Some of the macros are picking up local variables to save horizontal space,
11320	* (being able to see it in a table is the lesser evil here).
11321	*/
11322	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11323	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11324	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11325	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11326	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11327	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11328	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11329	} else do { } while (0)
11330	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11331	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11332	{ \
11333	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11334	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11335	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11336	} else do { } while (0)
11337	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11338	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11339	{ \
11340	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11341	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11342	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11343	} else do { } while (0)
11344	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11345	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11346	{ \
11347	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11348	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11349	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11350	} else do { } while (0)
11351
11352	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11353	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11354	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11355	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11356	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11357
11358	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11359	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11360	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11361	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11362	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11363	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11364	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11365	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11366	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11367	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11368	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11369	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11370	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11371	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11372	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11373	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11374	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11375	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11376	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11377	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11378	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11379	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11380	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11381	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11382	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11383	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11384	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11385	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11386	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11387	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11388	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11389	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11390	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11391	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11392	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11393	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11394
11395	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11396	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11397	{
11398	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11399	\| CPUMCTX_EXTRN_APIC_TPR);
11400	AssertRC(rc);
11401
11402	#if 0 /** @todo fix me */
11403	pDbgState->fClearCr0Mask = true;
11404	pDbgState->fClearCr4Mask = true;
11405	#endif
11406	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11407	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11408	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11409	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11410	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11411	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11412	require clearing here and in the loop if we start using it. */
11413	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11414	}
11415	else
11416	{
11417	if (pDbgState->fClearCr0Mask)
11418	{
11419	pDbgState->fClearCr0Mask = false;
11420	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11421	}
11422	if (pDbgState->fClearCr4Mask)
11423	{
11424	pDbgState->fClearCr4Mask = false;
11425	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11426	}
11427	}
11428	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11430
11431	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11432	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11433	{
11434	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11435	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11436	}
11437	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11438	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11439
11440	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11441	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11442	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11443	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11444	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11445	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11446	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11447	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11448	#if 0 /** @todo too slow, fix handler. */
11449	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11450	#endif
11451	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11452
11453	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11454	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11455	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11456	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11457	{
11458	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11459	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11460	}
11461	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11462	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11463	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11464	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11465
11466	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11467	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11468	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11469	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11470	{
11471	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11472	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11473	}
11474	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11475	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11476	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11477	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11478
11479	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11480	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11481	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11482	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11483	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11484	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11485	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11486	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11487	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11488	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11489	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11490	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11491	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11492	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11493	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11494	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11495	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11496	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11497	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11498	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11499	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11500	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11501
11502	#undef IS_EITHER_ENABLED
11503	#undef SET_ONLY_XBM_IF_EITHER_EN
11504	#undef SET_CPE1_XBM_IF_EITHER_EN
11505	#undef SET_CPEU_XBM_IF_EITHER_EN
11506	#undef SET_CPE2_XBM_IF_EITHER_EN
11507
11508	/*
11509	* Sanitize the control stuff.
11510	*/
11511	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11512	if (pDbgState->fCpe2Extra)
11513	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11514	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11515	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11516	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11517	{
11518	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11519	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11520	}
11521
11522	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11523	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11524	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11525	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11526	}
11527
11528
11529	/**
11530	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11531	* appropriate.
11532	*
11533	* The caller has checked the VM-exit against the
11534	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11535	* already, so we don't have to do that either.
11536	*
11537	* @returns Strict VBox status code (i.e. informational status codes too).
11538	* @param pVCpu The cross context virtual CPU structure.
11539	* @param pVmxTransient The VMX-transient structure.
11540	* @param uExitReason The VM-exit reason.
11541	*
11542	* @remarks The name of this function is displayed by dtrace, so keep it short
11543	* and to the point. No longer than 33 chars long, please.
11544	*/
11545	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11546	{
11547	/*
11548	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11549	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11550	*
11551	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11552	* does. Must add/change/remove both places. Same ordering, please.
11553	*
11554	* Added/removed events must also be reflected in the next section
11555	* where we dispatch dtrace events.
11556	*/
11557	bool fDtrace1 = false;
11558	bool fDtrace2 = false;
11559	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11560	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11561	uint32_t uEventArg = 0;
11562	#define SET_EXIT(a_EventSubName) \
11563	do { \
11564	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11565	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11566	} while (0)
11567	#define SET_BOTH(a_EventSubName) \
11568	do { \
11569	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11570	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11571	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11572	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11573	} while (0)
11574	switch (uExitReason)
11575	{
11576	case VMX_EXIT_MTF:
11577	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11578
11579	case VMX_EXIT_XCPT_OR_NMI:
11580	{
11581	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11582	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11583	{
11584	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11585	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11586	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11587	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11588	{
11589	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11590	{
11591	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11592	uEventArg = pVmxTransient->uExitIntErrorCode;
11593	}
11594	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11595	switch (enmEvent1)
11596	{
11597	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11598	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11599	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11600	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11601	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11602	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11603	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11604	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11605	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11606	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11607	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11608	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11609	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11610	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11611	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11612	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11613	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11614	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11615	default: break;
11616	}
11617	}
11618	else
11619	AssertFailed();
11620	break;
11621
11622	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11623	uEventArg = idxVector;
11624	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11625	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11626	break;
11627	}
11628	break;
11629	}
11630
11631	case VMX_EXIT_TRIPLE_FAULT:
11632	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11633	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11634	break;
11635	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11636	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11637	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11638	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11639	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11640
11641	/* Instruction specific VM-exits: */
11642	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11643	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11644	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11645	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11646	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11647	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11648	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11649	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11650	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11651	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11652	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11653	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11654	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11655	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11656	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11657	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11658	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11659	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11660	case VMX_EXIT_MOV_CRX:
11661	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11662	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11663	SET_BOTH(CRX_READ);
11664	else
11665	SET_BOTH(CRX_WRITE);
11666	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11667	break;
11668	case VMX_EXIT_MOV_DRX:
11669	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11670	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11671	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11672	SET_BOTH(DRX_READ);
11673	else
11674	SET_BOTH(DRX_WRITE);
11675	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11676	break;
11677	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11678	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11679	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11680	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11681	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11682	case VMX_EXIT_GDTR_IDTR_ACCESS:
11683	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11684	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11685	{
11686	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11687	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11688	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11689	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11690	}
11691	break;
11692
11693	case VMX_EXIT_LDTR_TR_ACCESS:
11694	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11695	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11696	{
11697	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11698	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11699	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11700	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11701	}
11702	break;
11703
11704	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11705	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11706	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11707	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11708	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11709	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11710	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11711	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11712	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11713	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11714	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11715
11716	/* Events that aren't relevant at this point. */
11717	case VMX_EXIT_EXT_INT:
11718	case VMX_EXIT_INT_WINDOW:
11719	case VMX_EXIT_NMI_WINDOW:
11720	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11721	case VMX_EXIT_PREEMPT_TIMER:
11722	case VMX_EXIT_IO_INSTR:
11723	break;
11724
11725	/* Errors and unexpected events. */
11726	case VMX_EXIT_INIT_SIGNAL:
11727	case VMX_EXIT_SIPI:
11728	case VMX_EXIT_IO_SMI:
11729	case VMX_EXIT_SMI:
11730	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11731	case VMX_EXIT_ERR_MSR_LOAD:
11732	case VMX_EXIT_ERR_MACHINE_CHECK:
11733	case VMX_EXIT_PML_FULL:
11734	case VMX_EXIT_VIRTUALIZED_EOI:
11735	break;
11736
11737	default:
11738	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11739	break;
11740	}
11741	#undef SET_BOTH
11742	#undef SET_EXIT
11743
11744	/*
11745	* Dtrace tracepoints go first. We do them here at once so we don't
11746	* have to copy the guest state saving and stuff a few dozen times.
11747	* Down side is that we've got to repeat the switch, though this time
11748	* we use enmEvent since the probes are a subset of what DBGF does.
11749	*/
11750	if (fDtrace1 \|\| fDtrace2)
11751	{
11752	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11753	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11754	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11755	switch (enmEvent1)
11756	{
11757	/** @todo consider which extra parameters would be helpful for each probe. */
11758	case DBGFEVENT_END: break;
11759	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11760	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11761	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11762	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11763	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11764	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11765	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11766	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11767	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11768	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11769	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11770	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11771	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11772	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11773	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11774	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11775	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11776	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11777	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11778	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11779	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11780	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11781	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11782	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11783	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11784	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11785	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11786	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11787	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11788	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11789	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11790	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11791	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11792	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11793	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11794	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11795	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11796	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11797	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11798	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11799	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11800	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11801	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11802	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11803	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11804	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11805	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11806	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11807	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11808	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11809	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11810	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11811	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11812	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11813	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11814	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11815	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11816	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11817	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11818	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11819	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11820	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11821	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11822	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11823	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11824	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11825	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11826	}
11827	switch (enmEvent2)
11828	{
11829	/** @todo consider which extra parameters would be helpful for each probe. */
11830	case DBGFEVENT_END: break;
11831	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11832	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11833	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11834	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11835	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11836	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11837	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11838	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11839	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11840	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11841	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11842	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11843	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11844	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11845	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11846	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11847	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11848	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11849	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11850	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11851	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11852	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11853	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11854	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11855	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11856	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11857	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11858	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11859	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11860	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11861	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11862	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11863	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11864	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11865	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11866	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11867	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11868	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11869	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11870	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11871	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11872	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11873	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11874	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11875	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11876	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11877	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11878	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11879	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11880	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11881	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11882	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11883	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11884	}
11885	}
11886
11887	/*
11888	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11889	* the DBGF call will do a full check).
11890	*
11891	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11892	* Note! If we have to events, we prioritize the first, i.e. the instruction
11893	* one, in order to avoid event nesting.
11894	*/
11895	PVM pVM = pVCpu->CTX_SUFF(pVM);
11896	if ( enmEvent1 != DBGFEVENT_END
11897	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11898	{
11899	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11900	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11901	if (rcStrict != VINF_SUCCESS)
11902	return rcStrict;
11903	}
11904	else if ( enmEvent2 != DBGFEVENT_END
11905	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11906	{
11907	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11908	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11909	if (rcStrict != VINF_SUCCESS)
11910	return rcStrict;
11911	}
11912
11913	return VINF_SUCCESS;
11914	}
11915
11916
11917	/**
11918	* Single-stepping VM-exit filtering.
11919	*
11920	* This is preprocessing the VM-exits and deciding whether we've gotten far
11921	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11922	* handling is performed.
11923	*
11924	* @returns Strict VBox status code (i.e. informational status codes too).
11925	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11926	* @param pVmxTransient The VMX-transient structure.
11927	* @param pDbgState The debug state.
11928	*/
11929	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11930	{
11931	/*
11932	* Expensive (saves context) generic dtrace VM-exit probe.
11933	*/
11934	uint32_t const uExitReason = pVmxTransient->uExitReason;
11935	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11936	{ /* more likely */ }
11937	else
11938	{
11939	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11940	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11941	AssertRC(rc);
11942	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11943	}
11944
11945	/*
11946	* Check for host NMI, just to get that out of the way.
11947	*/
11948	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11949	{ /* normally likely */ }
11950	else
11951	{
11952	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11953	AssertRCReturn(rc2, rc2);
11954	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11955	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11956	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11957	}
11958
11959	/*
11960	* Check for single stepping event if we're stepping.
11961	*/
11962	if (pVCpu->hm.s.fSingleInstruction)
11963	{
11964	switch (uExitReason)
11965	{
11966	case VMX_EXIT_MTF:
11967	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11968
11969	/* Various events: */
11970	case VMX_EXIT_XCPT_OR_NMI:
11971	case VMX_EXIT_EXT_INT:
11972	case VMX_EXIT_TRIPLE_FAULT:
11973	case VMX_EXIT_INT_WINDOW:
11974	case VMX_EXIT_NMI_WINDOW:
11975	case VMX_EXIT_TASK_SWITCH:
11976	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11977	case VMX_EXIT_APIC_ACCESS:
11978	case VMX_EXIT_EPT_VIOLATION:
11979	case VMX_EXIT_EPT_MISCONFIG:
11980	case VMX_EXIT_PREEMPT_TIMER:
11981
11982	/* Instruction specific VM-exits: */
11983	case VMX_EXIT_CPUID:
11984	case VMX_EXIT_GETSEC:
11985	case VMX_EXIT_HLT:
11986	case VMX_EXIT_INVD:
11987	case VMX_EXIT_INVLPG:
11988	case VMX_EXIT_RDPMC:
11989	case VMX_EXIT_RDTSC:
11990	case VMX_EXIT_RSM:
11991	case VMX_EXIT_VMCALL:
11992	case VMX_EXIT_VMCLEAR:
11993	case VMX_EXIT_VMLAUNCH:
11994	case VMX_EXIT_VMPTRLD:
11995	case VMX_EXIT_VMPTRST:
11996	case VMX_EXIT_VMREAD:
11997	case VMX_EXIT_VMRESUME:
11998	case VMX_EXIT_VMWRITE:
11999	case VMX_EXIT_VMXOFF:
12000	case VMX_EXIT_VMXON:
12001	case VMX_EXIT_MOV_CRX:
12002	case VMX_EXIT_MOV_DRX:
12003	case VMX_EXIT_IO_INSTR:
12004	case VMX_EXIT_RDMSR:
12005	case VMX_EXIT_WRMSR:
12006	case VMX_EXIT_MWAIT:
12007	case VMX_EXIT_MONITOR:
12008	case VMX_EXIT_PAUSE:
12009	case VMX_EXIT_GDTR_IDTR_ACCESS:
12010	case VMX_EXIT_LDTR_TR_ACCESS:
12011	case VMX_EXIT_INVEPT:
12012	case VMX_EXIT_RDTSCP:
12013	case VMX_EXIT_INVVPID:
12014	case VMX_EXIT_WBINVD:
12015	case VMX_EXIT_XSETBV:
12016	case VMX_EXIT_RDRAND:
12017	case VMX_EXIT_INVPCID:
12018	case VMX_EXIT_VMFUNC:
12019	case VMX_EXIT_RDSEED:
12020	case VMX_EXIT_XSAVES:
12021	case VMX_EXIT_XRSTORS:
12022	{
12023	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12024	AssertRCReturn(rc, rc);
12025	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12026	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12027	return VINF_EM_DBG_STEPPED;
12028	break;
12029	}
12030
12031	/* Errors and unexpected events: */
12032	case VMX_EXIT_INIT_SIGNAL:
12033	case VMX_EXIT_SIPI:
12034	case VMX_EXIT_IO_SMI:
12035	case VMX_EXIT_SMI:
12036	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12037	case VMX_EXIT_ERR_MSR_LOAD:
12038	case VMX_EXIT_ERR_MACHINE_CHECK:
12039	case VMX_EXIT_PML_FULL:
12040	case VMX_EXIT_VIRTUALIZED_EOI:
12041	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12042	break;
12043
12044	default:
12045	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12046	break;
12047	}
12048	}
12049
12050	/*
12051	* Check for debugger event breakpoints and dtrace probes.
12052	*/
12053	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12054	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12055	{
12056	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12057	if (rcStrict != VINF_SUCCESS)
12058	return rcStrict;
12059	}
12060
12061	/*
12062	* Normal processing.
12063	*/
12064	#ifdef HMVMX_USE_FUNCTION_TABLE
12065	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12066	#else
12067	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12068	#endif
12069	}
12070
12071
12072	/**
12073	* Single steps guest code using hardware-assisted VMX.
12074	*
12075	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12076	* but single-stepping through the hypervisor debugger.
12077	*
12078	* @returns Strict VBox status code (i.e. informational status codes too).
12079	* @param pVCpu The cross context virtual CPU structure.
12080	* @param pcLoops Pointer to the number of executed loops.
12081	*
12082	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12083	*/
12084	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12085	{
12086	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12087	Assert(pcLoops);
12088	Assert(*pcLoops <= cMaxResumeLoops);
12089
12090	VMXTRANSIENT VmxTransient;
12091	RT_ZERO(VmxTransient);
12092	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12093
12094	/* Set HMCPU indicators. */
12095	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12096	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12097	pVCpu->hm.s.fDebugWantRdTscExit = false;
12098	pVCpu->hm.s.fUsingDebugLoop = true;
12099
12100	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12101	VMXRUNDBGSTATE DbgState;
12102	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12103	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12104
12105	/*
12106	* The loop.
12107	*/
12108	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12109	for (;;)
12110	{
12111	Assert(!HMR0SuspendPending());
12112	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12113	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12114	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12115
12116	/* Set up VM-execution controls the next two can respond to. */
12117	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12118
12119	/*
12120	* Preparatory work for running guest code, this may force us to
12121	* return to ring-3.
12122	*
12123	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12124	*/
12125	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12126	if (rcStrict != VINF_SUCCESS)
12127	break;
12128
12129	/* Interrupts are disabled at this point! */
12130	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12131
12132	/* Override any obnoxious code in the above two calls. */
12133	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12134
12135	/*
12136	* Finally execute the guest.
12137	*/
12138	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12139
12140	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12141	/* Interrupts are re-enabled at this point! */
12142
12143	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12144	if (RT_SUCCESS(rcRun))
12145	{ /* very likely */ }
12146	else
12147	{
12148	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12149	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12150	return rcRun;
12151	}
12152
12153	/* Profile the VM-exit. */
12154	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12156	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12157	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12158	HMVMX_START_EXIT_DISPATCH_PROF();
12159
12160	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12161
12162	/*
12163	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12164	*/
12165	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12166	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12167	if (rcStrict != VINF_SUCCESS)
12168	break;
12169	if (++(*pcLoops) > cMaxResumeLoops)
12170	{
12171	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12172	rcStrict = VINF_EM_RAW_INTERRUPT;
12173	break;
12174	}
12175
12176	/*
12177	* Stepping: Did the RIP change, if so, consider it a single step.
12178	* Otherwise, make sure one of the TFs gets set.
12179	*/
12180	if (fStepping)
12181	{
12182	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12183	AssertRC(rc);
12184	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12185	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12186	{
12187	rcStrict = VINF_EM_DBG_STEPPED;
12188	break;
12189	}
12190	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12191	}
12192
12193	/*
12194	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12195	*/
12196	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12197	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12198	}
12199
12200	/*
12201	* Clear the X86_EFL_TF if necessary.
12202	*/
12203	if (pVCpu->hm.s.fClearTrapFlag)
12204	{
12205	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12206	AssertRC(rc);
12207	pVCpu->hm.s.fClearTrapFlag = false;
12208	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12209	}
12210	/** @todo there seems to be issues with the resume flag when the monitor trap
12211	* flag is pending without being used. Seen early in bios init when
12212	* accessing APIC page in protected mode. */
12213
12214	/*
12215	* Restore VM-exit control settings as we may not re-enter this function the
12216	* next time around.
12217	*/
12218	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12219
12220	/* Restore HMCPU indicators. */
12221	pVCpu->hm.s.fUsingDebugLoop = false;
12222	pVCpu->hm.s.fDebugWantRdTscExit = false;
12223	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12224
12225	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12226	return rcStrict;
12227	}
12228
12229
12230	/** @} */
12231
12232
12233	/**
12234	* Checks if any expensive dtrace probes are enabled and we should go to the
12235	* debug loop.
12236	*
12237	* @returns true if we should use debug loop, false if not.
12238	*/
12239	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12240	{
12241	/* It's probably faster to OR the raw 32-bit counter variables together.
12242	Since the variables are in an array and the probes are next to one
12243	another (more or less), we have good locality. So, better read
12244	eight-nine cache lines ever time and only have one conditional, than
12245	128+ conditionals, right? */
12246	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12247	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12248	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12249	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12250	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12251	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12252	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12253	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12254	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12255	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12256	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12257	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12258	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12259	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12260	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12261	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12262	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12263	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12264	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12265	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12266	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12267	) != 0
12268	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12269	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12270	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12271	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12272	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12273	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12274	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12275	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12276	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12277	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12278	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12279	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12280	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12281	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12282	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12283	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12284	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12285	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12286	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12287	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12288	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12289	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12290	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12291	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12292	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12293	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12294	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12295	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12296	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12297	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12298	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12299	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12300	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12301	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12302	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12303	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12304	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12305	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12306	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12307	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12308	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12309	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12310	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12311	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12312	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12313	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12314	) != 0
12315	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12316	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12317	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12318	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12319	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12320	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12321	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12322	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12323	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12324	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12325	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12326	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12327	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12328	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12329	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12330	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12331	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12332	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12333	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12334	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12335	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12336	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12337	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12338	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12339	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12340	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12341	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12342	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12343	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12344	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12345	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12346	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12347	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12348	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12349	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12350	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12351	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12352	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12353	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12354	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12355	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12356	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12357	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12358	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12359	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12360	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12361	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12362	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12363	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12364	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12365	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12366	) != 0;
12367	}
12368
12369
12370	/**
12371	* Runs the guest using hardware-assisted VMX.
12372	*
12373	* @returns Strict VBox status code (i.e. informational status codes too).
12374	* @param pVCpu The cross context virtual CPU structure.
12375	*/
12376	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12377	{
12378	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12379	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12380	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12381	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12382
12383	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12384
12385	VBOXSTRICTRC rcStrict;
12386	uint32_t cLoops = 0;
12387	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12388	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12389	#else
12390	bool const fInNestedGuestMode = false;
12391	#endif
12392	if (!fInNestedGuestMode)
12393	{
12394	if ( !pVCpu->hm.s.fUseDebugLoop
12395	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12396	&& !DBGFIsStepping(pVCpu)
12397	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12398	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12399	else
12400	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12401	}
12402	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12403	else
12404	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12405
12406	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12407	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12408	#endif
12409
12410	if (rcStrict == VERR_EM_INTERPRETER)
12411	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12412	else if (rcStrict == VINF_EM_RESET)
12413	rcStrict = VINF_EM_TRIPLE_FAULT;
12414
12415	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12416	if (RT_FAILURE(rc2))
12417	{
12418	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12419	rcStrict = rc2;
12420	}
12421	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12422	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12423	return rcStrict;
12424	}
12425
12426
12427	#ifndef HMVMX_USE_FUNCTION_TABLE
12428	/**
12429	* Handles a guest VM-exit from hardware-assisted VMX execution.
12430	*
12431	* @returns Strict VBox status code (i.e. informational status codes too).
12432	* @param pVCpu The cross context virtual CPU structure.
12433	* @param pVmxTransient The VMX-transient structure.
12434	*/
12435	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12436	{
12437	#ifdef DEBUG_ramshankar
12438	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12439	do { \
12440	if (a_fSave != 0) \
12441	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12442	VBOXSTRICTRC rcStrict = a_CallExpr; \
12443	if (a_fSave != 0) \
12444	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12445	return rcStrict; \
12446	} while (0)
12447	#else
12448	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12449	#endif
12450	uint32_t const rcReason = pVmxTransient->uExitReason;
12451	switch (rcReason)
12452	{
12453	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12454	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12455	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12456	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12457	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12458	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12459	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12460	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12461	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12462	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12463	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12464	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12465	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12466	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12467	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12468	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12469	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12470	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12471	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12472	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12473	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12474	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12475	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12476	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12477	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12478	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12479	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12480	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12481	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12482	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12483	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12484	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12485	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12486	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12487	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12488	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12489	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12490	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12491	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12492	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12493	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12494	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12495	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12496	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12497	#else
12498	case VMX_EXIT_VMCLEAR:
12499	case VMX_EXIT_VMLAUNCH:
12500	case VMX_EXIT_VMPTRLD:
12501	case VMX_EXIT_VMPTRST:
12502	case VMX_EXIT_VMREAD:
12503	case VMX_EXIT_VMRESUME:
12504	case VMX_EXIT_VMWRITE:
12505	case VMX_EXIT_VMXOFF:
12506	case VMX_EXIT_VMXON:
12507	case VMX_EXIT_INVVPID:
12508	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12509	#endif
12510
12511	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12512	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12513	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12514
12515	case VMX_EXIT_RSM:
12516	case VMX_EXIT_RDSEED:
12517	case VMX_EXIT_ENCLS:
12518	case VMX_EXIT_INVEPT:
12519	case VMX_EXIT_VMFUNC:
12520	case VMX_EXIT_XSAVES:
12521	case VMX_EXIT_XRSTORS:
12522	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12523
12524	case VMX_EXIT_INIT_SIGNAL:
12525	case VMX_EXIT_SIPI:
12526	case VMX_EXIT_IO_SMI:
12527	case VMX_EXIT_SMI:
12528	case VMX_EXIT_ERR_MSR_LOAD:
12529	case VMX_EXIT_ERR_MACHINE_CHECK:
12530	case VMX_EXIT_PML_FULL:
12531	case VMX_EXIT_VIRTUALIZED_EOI:
12532	case VMX_EXIT_APIC_WRITE:
12533	default:
12534	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12535	}
12536	#undef VMEXIT_CALL_RET
12537	}
12538	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12539
12540
12541	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12542	/**
12543	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12544	*
12545	* @returns Strict VBox status code (i.e. informational status codes too).
12546	* @param pVCpu The cross context virtual CPU structure.
12547	* @param pVmxTransient The VMX-transient structure.
12548	*/
12549	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12550	{
12551	VBOXSTRICTRC rcStrict;
12552	uint32_t const uExitReason = pVmxTransient->uExitReason;
12553	switch (uExitReason)
12554	{
12555	case VMX_EXIT_EPT_MISCONFIG:
12556	rcStrict = hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12557	break;
12558
12559	case VMX_EXIT_EPT_VIOLATION:
12560	rcStrict = hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12561	break;
12562
12563	case VMX_EXIT_IO_INSTR:
12564	{
12565	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12566	AssertRCReturn(rc, rc);
12567
12568	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12569	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12570	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12571
12572	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
12573	uint8_t const cbAccess = s_aIOSizes[uIOSize];
12574	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
12575	{
12576	/*
12577	* IN/OUT instruction:
12578	* - Provides VM-exit instruction length.
12579	*
12580	* INS/OUTS instruction:
12581	* - Provides VM-exit instruction length.
12582	* - Provides Guest-linear address.
12583	* - Optionally provides VM-exit instruction info (depends on CPU feature).
12584	*/
12585	PVM pVM = pVCpu->CTX_SUFF(pVM);
12586	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12587	AssertRCReturn(rc, rc);
12588
12589	/* Make sure we don't use stale VMX-transient info. */
12590	pVmxTransient->ExitInstrInfo.u = 0;
12591	pVmxTransient->uGuestLinearAddr = 0;
12592
12593	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
12594	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12595	if (fIOString)
12596	{
12597	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12598	if (fVmxInsOutsInfo)
12599	{
12600	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
12601	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12602	}
12603	}
12604	AssertRCReturn(rc, rc);
12605
12606	VMXVEXITINFO ExitInfo;
12607	RT_ZERO(ExitInfo);
12608	ExitInfo.uReason = uExitReason;
12609	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12610	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12611	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12612	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
12613	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12614	}
12615	else
12616	rcStrict = hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
12617	break;
12618	}
12619
12620	case VMX_EXIT_HLT:
12621	{
12622	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12623	AssertRCReturn(rc, rc);
12624	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
12625	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12626	else
12627	rcStrict = hmR0VmxExitHlt(pVCpu, pVmxTransient);
12628	break;
12629	}
12630
12631	case VMX_EXIT_RDTSC:
12632	{
12633	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12634	AssertRCReturn(rc, rc);
12635	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12636	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12637	else
12638	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12639	break;
12640	}
12641
12642	case VMX_EXIT_RDTSCP:
12643	{
12644	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12645	AssertRCReturn(rc, rc);
12646	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12647	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12648	else
12649	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12650	break;
12651	}
12652
12653	/*
12654	* Instructions that cause VM-exits unconditionally.
12655	* - Provides VM-exit instruction length ONLY.
12656	*/
12657	case VMX_EXIT_CPUID:
12658	case VMX_EXIT_VMCALL:
12659	case VMX_EXIT_GETSEC:
12660	case VMX_EXIT_INVD:
12661	case VMX_EXIT_XSETBV:
12662	case VMX_EXIT_VMLAUNCH:
12663	case VMX_EXIT_VMRESUME:
12664	case VMX_EXIT_VMXOFF:
12665	{
12666	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12667	AssertRCReturn(rc, rc);
12668	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12669	break;
12670	}
12671
12672	/*
12673	* Instructions that cause VM-exits unconditionally.
12674	* - Provides VM-exit instruction length.
12675	* - Provides VM-exit information.
12676	* - Optionally provides VM-exit qualification.
12677	*
12678	* Since VM-exit qualification is 0 for all VM-exits where it is not
12679	* applicable, reading and passing it to the guest should produce
12680	* defined behavior.
12681	*
12682	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12683	*/
12684	case VMX_EXIT_INVEPT:
12685	case VMX_EXIT_INVVPID:
12686	case VMX_EXIT_VMCLEAR:
12687	case VMX_EXIT_VMPTRLD:
12688	case VMX_EXIT_VMPTRST:
12689	case VMX_EXIT_VMXON:
12690	{
12691	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12692	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12693	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12694	AssertRCReturn(rc, rc);
12695
12696	VMXVEXITINFO ExitInfo;
12697	RT_ZERO(ExitInfo);
12698	ExitInfo.uReason = uExitReason;
12699	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12700	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12701	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12702	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12703	break;
12704	}
12705
12706	case VMX_EXIT_INVLPG:
12707	{
12708	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12709	{
12710	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12711	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12712	AssertRCReturn(rc, rc);
12713
12714	VMXVEXITINFO ExitInfo;
12715	RT_ZERO(ExitInfo);
12716	ExitInfo.uReason = uExitReason;
12717	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12718	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12719	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12720	}
12721	else
12722	rcStrict = hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
12723	break;
12724	}
12725
12726	case VMX_EXIT_INVPCID:
12727	{
12728	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12729	{
12730	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12731	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12732	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12733	AssertRCReturn(rc, rc);
12734
12735	VMXVEXITINFO ExitInfo;
12736	RT_ZERO(ExitInfo);
12737	ExitInfo.uReason = uExitReason;
12738	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12739	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12740	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12741	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12742	}
12743	else
12744	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12745	break;
12746	}
12747
12748	case VMX_EXIT_RDMSR:
12749	{
12750	uint32_t fMsrpm;
12751	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12752	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12753	else
12754	fMsrpm = VMXMSRPM_EXIT_RD;
12755
12756	if (fMsrpm & VMXMSRPM_EXIT_RD)
12757	{
12758	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12759	AssertRCReturn(rc, rc);
12760	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12761	}
12762	else
12763	rcStrict = hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
12764	break;
12765	}
12766
12767	case VMX_EXIT_WRMSR:
12768	{
12769	uint32_t fMsrpm;
12770	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12771	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12772	else
12773	fMsrpm = VMXMSRPM_EXIT_WR;
12774
12775	if (fMsrpm & VMXMSRPM_EXIT_WR)
12776	{
12777	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12778	AssertRCReturn(rc, rc);
12779	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12780	}
12781	else
12782	rcStrict = hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
12783	break;
12784	}
12785
12786	case VMX_EXIT_TASK_SWITCH:
12787	{
12788	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12789	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12790	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12791	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12792	AssertRCReturn(rc, rc);
12793
12794	VMXVEXITINFO ExitInfo;
12795	RT_ZERO(ExitInfo);
12796	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12797	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12798
12799	VMXVEXITEVENTINFO ExitEventInfo;
12800	RT_ZERO(ExitInfo);
12801	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
12802	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
12803
12804	rcStrict = IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
12805	break;
12806	}
12807
12808	case VMX_EXIT_WBINVD:
12809	{
12810	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
12811	{
12812	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12813	AssertRCReturn(rc, rc);
12814	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12815	}
12816	else
12817	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12818	break;
12819	}
12820
12821	case VMX_EXIT_MTF:
12822	{
12823	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
12824	rcStrict = IEMExecVmxVmexit(pVCpu, uExitReason);
12825	break;
12826	}
12827
12828	case VMX_EXIT_APIC_ACCESS:
12829	case VMX_EXIT_XCPT_OR_NMI:
12830	{
12831	/** @todo NSTVMX: APIC-access, Xcpt or NMI, Mov CRx. */
12832	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12833	break;
12834	}
12835
12836	case VMX_EXIT_MOV_CRX:
12837	{
12838	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12839	AssertRCReturn(rc, rc);
12840
12841	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
12842	switch (uAccessType)
12843	{
12844	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
12845	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
12846	{
12847	/** @todo NSTVMX: Implement me. */
12848	break;
12849	}
12850
12851	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
12852	{
12853	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
12854	Assert(pVmcsNstGst);
12855
12856	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
12857	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
12858	if ( (uGstHostMask & X86_CR0_TS)
12859	&& (uReadShadow & X86_CR0_TS))
12860	{
12861	/** @todo NSTVMX: continue with VM-exit. */
12862	}
12863	break;
12864	}
12865
12866	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12867	{
12868	RTGCPTR GCPtrEffDst;
12869	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
12870	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
12871
12872	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12873	if (fMemOperand)
12874	{
12875	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12876	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
12877	}
12878	else
12879	GCPtrEffDst = NIL_RTGCPTR;
12880	AssertRCReturn(rc, rc);
12881
12882	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
12883	{
12884	VMXVEXITINFO ExitInfo;
12885	RT_ZERO(ExitInfo);
12886	ExitInfo.uReason = VMX_EXIT_MOV_CRX;
12887	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12888	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
12889	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12890	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12891	}
12892	else
12893	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
12894	break;
12895	}
12896
12897	default:
12898	{
12899	pVCpu->hm.s.u32HMError = uAccessType;
12900	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12901	VERR_VMX_UNEXPECTED_EXCEPTION);
12902	}
12903	}
12904	break;
12905	}
12906
12907	case VMX_EXIT_EXT_INT:
12908	{
12909	/* We shouldn't direct physical interrupts to the nested-guest. */
12910	rcStrict = hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12911	break;
12912	}
12913
12914	case VMX_EXIT_INT_WINDOW:
12915	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12916	{
12917	/** @todo NSTVMX: Interrupt window, TPR below threshold. */
12918	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12919	break;
12920	}
12921
12922	case VMX_EXIT_MWAIT:
12923	{
12924	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
12925	{
12926	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12927	AssertRCReturn(rc, rc);
12928	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12929	}
12930	else
12931	rcStrict = hmR0VmxExitMwait(pVCpu, pVmxTransient);
12932	break;
12933	}
12934
12935	case VMX_EXIT_MONITOR:
12936	{
12937	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
12938	{
12939	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12940	AssertRCReturn(rc, rc);
12941	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12942	}
12943	else
12944	rcStrict = hmR0VmxExitMonitor(pVCpu, pVmxTransient);
12945	break;
12946	}
12947
12948	case VMX_EXIT_PAUSE:
12949	{
12950	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
12951	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
12952	* PAUSE when executing a nested-guest? If it does not, we would not need
12953	* to check for the intercepts here. Just call VM-exit... */
12954	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
12955	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
12956	{
12957	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12958	AssertRCReturn(rc, rc);
12959	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12960	}
12961	else
12962	rcStrict = hmR0VmxExitPause(pVCpu, pVmxTransient);
12963	break;
12964	}
12965
12966	case VMX_EXIT_PREEMPT_TIMER:
12967	{
12968	/** @todo NSTVMX: Preempt timer. */
12969	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12970	break;
12971	}
12972
12973	case VMX_EXIT_MOV_DRX:
12974	{
12975	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
12976	{
12977	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12978	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12979	AssertRCReturn(rc, rc);
12980
12981	VMXVEXITINFO ExitInfo;
12982	RT_ZERO(ExitInfo);
12983	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12984	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12985	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12986	}
12987	else
12988	rcStrict = hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
12989	break;
12990	}
12991
12992	case VMX_EXIT_GDTR_IDTR_ACCESS:
12993	case VMX_EXIT_LDTR_TR_ACCESS:
12994	{
12995	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
12996	{
12997	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12998	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12999	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13000	AssertRCReturn(rc, rc);
13001
13002	VMXVEXITINFO ExitInfo;
13003	RT_ZERO(ExitInfo);
13004	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13005	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13006	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
13007	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13008	}
13009	else
13010	rcStrict = hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient);
13011	break;
13012	}
13013
13014	case VMX_EXIT_RDRAND:
13015	case VMX_EXIT_RDPMC:
13016	case VMX_EXIT_VMREAD:
13017	case VMX_EXIT_VMWRITE:
13018	case VMX_EXIT_RSM:
13019	case VMX_EXIT_RDSEED:
13020	case VMX_EXIT_ENCLS:
13021	case VMX_EXIT_VMFUNC:
13022	case VMX_EXIT_XSAVES:
13023	case VMX_EXIT_XRSTORS:
13024
13025	case VMX_EXIT_TRIPLE_FAULT:
13026	case VMX_EXIT_NMI_WINDOW:
13027	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
13028
13029	case VMX_EXIT_INIT_SIGNAL:
13030	case VMX_EXIT_SIPI:
13031	case VMX_EXIT_IO_SMI:
13032	case VMX_EXIT_SMI:
13033	case VMX_EXIT_ERR_MSR_LOAD:
13034	case VMX_EXIT_ERR_MACHINE_CHECK:
13035	case VMX_EXIT_PML_FULL:
13036	case VMX_EXIT_VIRTUALIZED_EOI:
13037	case VMX_EXIT_APIC_WRITE:
13038	default:
13039	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13040	break;
13041	}
13042
13043	return rcStrict;
13044	}
13045	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13046
13047
13048	#ifdef VBOX_STRICT
13049	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13050	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13051	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13052
13053	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13054	do { \
13055	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13056	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13057	} while (0)
13058
13059	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13060	do { \
13061	AssertPtr((a_pVCpu)); \
13062	AssertPtr((a_pVmxTransient)); \
13063	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13064	Assert((a_pVmxTransient)->pVmcsInfo); \
13065	Assert(ASMIntAreEnabled()); \
13066	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13067	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13068	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
13069	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13070	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13071	HMVMX_ASSERT_PREEMPT_CPUID(); \
13072	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13073	} while (0)
13074
13075	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13076	do { \
13077	Log4Func(("\n")); \
13078	} while (0)
13079	#else
13080	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13081	do { \
13082	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13083	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13084	} while (0)
13085	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13086	#endif
13087
13088
13089	/**
13090	* Advances the guest RIP by the specified number of bytes.
13091	*
13092	* @param pVCpu The cross context virtual CPU structure.
13093	* @param cbInstr Number of bytes to advance the RIP by.
13094	*
13095	* @remarks No-long-jump zone!!!
13096	*/
13097	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13098	{
13099	/* Advance the RIP. */
13100	pVCpu->cpum.GstCtx.rip += cbInstr;
13101	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13102
13103	/* Update interrupt inhibition. */
13104	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13105	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13106	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13107	}
13108
13109
13110	/**
13111	* Advances the guest RIP after reading it from the VMCS.
13112	*
13113	* @returns VBox status code, no informational status codes.
13114	* @param pVCpu The cross context virtual CPU structure.
13115	* @param pVmxTransient The VMX-transient structure.
13116	*
13117	* @remarks No-long-jump zone!!!
13118	*/
13119	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13120	{
13121	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13122	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13123	AssertRCReturn(rc, rc);
13124
13125	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13126	return VINF_SUCCESS;
13127	}
13128
13129
13130	/**
13131	* Handle a condition that occurred while delivering an event through the guest
13132	* IDT.
13133	*
13134	* @returns Strict VBox status code (i.e. informational status codes too).
13135	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13136	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13137	* to continue execution of the guest which will delivery the \#DF.
13138	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13139	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13140	*
13141	* @param pVCpu The cross context virtual CPU structure.
13142	* @param pVmxTransient The VMX-transient structure.
13143	*
13144	* @remarks No-long-jump zone!!!
13145	*/
13146	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13147	{
13148	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13149
13150	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13151	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13152	AssertRCReturn(rc2, rc2);
13153
13154	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13155	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13156	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13157	{
13158	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13159	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13160
13161	/*
13162	* If the event was a software interrupt (generated with INT n) or a software exception
13163	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13164	* can handle the VM-exit and continue guest execution which will re-execute the
13165	* instruction rather than re-injecting the exception, as that can cause premature
13166	* trips to ring-3 before injection and involve TRPM which currently has no way of
13167	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13168	* the problem).
13169	*/
13170	IEMXCPTRAISE enmRaise;
13171	IEMXCPTRAISEINFO fRaiseInfo;
13172	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13173	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13174	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13175	{
13176	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13177	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13178	}
13179	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13180	{
13181	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13182	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13183	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13184	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13185	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13186	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13187
13188	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13189
13190	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13191	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13192	{
13193	pVmxTransient->fVectoringPF = true;
13194	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13195	}
13196	}
13197	else
13198	{
13199	/*
13200	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13201	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13202	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13203	*/
13204	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13205	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13206	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13207	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13208	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13209	}
13210
13211	/*
13212	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13213	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13214	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13215	* subsequent VM-entry would fail.
13216	*
13217	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13218	*/
13219	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
13220	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13221	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13222	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13223	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13224	{
13225	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
13226	}
13227
13228	switch (enmRaise)
13229	{
13230	case IEMXCPTRAISE_CURRENT_XCPT:
13231	{
13232	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13233	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13234	Assert(rcStrict == VINF_SUCCESS);
13235	break;
13236	}
13237
13238	case IEMXCPTRAISE_PREV_EVENT:
13239	{
13240	uint32_t u32ErrCode;
13241	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13242	{
13243	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13244	AssertRCReturn(rc2, rc2);
13245	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13246	}
13247	else
13248	u32ErrCode = 0;
13249
13250	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13251	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13252	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13253	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13254
13255	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13256	pVCpu->hm.s.Event.u32ErrCode));
13257	Assert(rcStrict == VINF_SUCCESS);
13258	break;
13259	}
13260
13261	case IEMXCPTRAISE_REEXEC_INSTR:
13262	Assert(rcStrict == VINF_SUCCESS);
13263	break;
13264
13265	case IEMXCPTRAISE_DOUBLE_FAULT:
13266	{
13267	/*
13268	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13269	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13270	*/
13271	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13272	{
13273	pVmxTransient->fVectoringDoublePF = true;
13274	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13275	pVCpu->cpum.GstCtx.cr2));
13276	rcStrict = VINF_SUCCESS;
13277	}
13278	else
13279	{
13280	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13281	hmR0VmxSetPendingXcptDF(pVCpu);
13282	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13283	uIdtVector, uExitVector));
13284	rcStrict = VINF_HM_DOUBLE_FAULT;
13285	}
13286	break;
13287	}
13288
13289	case IEMXCPTRAISE_TRIPLE_FAULT:
13290	{
13291	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13292	rcStrict = VINF_EM_RESET;
13293	break;
13294	}
13295
13296	case IEMXCPTRAISE_CPU_HANG:
13297	{
13298	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13299	rcStrict = VERR_EM_GUEST_CPU_HANG;
13300	break;
13301	}
13302
13303	default:
13304	{
13305	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13306	rcStrict = VERR_VMX_IPE_2;
13307	break;
13308	}
13309	}
13310	}
13311	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13312	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13313	&& uExitVector != X86_XCPT_DF
13314	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13315	{
13316	/*
13317	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13318	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13319	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13320	*/
13321	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
13322	{
13323	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
13324	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
13325	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
13326	}
13327	}
13328
13329	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13330	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13331	return rcStrict;
13332	}
13333
13334
13335	/** @name VM-exit handlers.
13336	* @{
13337	*/
13338	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13339	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13340	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13341
13342	/**
13343	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13344	*/
13345	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13346	{
13347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13349	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13350	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13351	return VINF_SUCCESS;
13352	return VINF_EM_RAW_INTERRUPT;
13353	}
13354
13355
13356	/**
13357	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13358	*/
13359	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13360	{
13361	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13362	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13363
13364	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13365	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13366	AssertRCReturn(rc, rc);
13367
13368	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13369	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13370	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13371	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13372
13373	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13374	{
13375	/*
13376	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13377	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13378	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13379	*
13380	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13381	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13382	*/
13383	VMXDispatchHostNmi();
13384	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
13385	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13386	return VINF_SUCCESS;
13387	}
13388
13389	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13390	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13391	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
13392	{ /* likely */ }
13393	else
13394	{
13395	if (rcStrict == VINF_HM_DOUBLE_FAULT)
13396	rcStrict = VINF_SUCCESS;
13397	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13398	return rcStrict;
13399	}
13400
13401	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13402	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13403	switch (uIntType)
13404	{
13405	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13406	Assert(uVector == X86_XCPT_DB);
13407	RT_FALL_THRU();
13408	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13409	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13410	RT_FALL_THRU();
13411	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13412	{
13413	/*
13414	* If there's any exception caused as a result of event injection, the resulting
13415	* secondary/final execption will be pending, we shall continue guest execution
13416	* after injecting the event. The page-fault case is complicated and we manually
13417	* handle any currently pending event in hmR0VmxExitXcptPF.
13418	*/
13419	if (!pVCpu->hm.s.Event.fPending)
13420	{ /* likely */ }
13421	else if (uVector != X86_XCPT_PF)
13422	{
13423	rcStrict = VINF_SUCCESS;
13424	break;
13425	}
13426
13427	switch (uVector)
13428	{
13429	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13430	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13431	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13432	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13433	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13434	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13435
13436	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13437	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13438	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13439	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13440	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13441	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13442	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13443	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13444	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13445	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13446	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13447	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13448	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13449	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13450	default:
13451	{
13452	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13453	if (pVmcsInfo->RealMode.fRealOnV86Active)
13454	{
13455	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13456	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13457	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13458
13459	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13460	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13461	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13462	AssertRCReturn(rc, rc);
13463	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13464	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13465	0 /* GCPtrFaultAddress */);
13466	rcStrict = VINF_SUCCESS;
13467	}
13468	else
13469	{
13470	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13471	pVCpu->hm.s.u32HMError = uVector;
13472	rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
13473	}
13474	break;
13475	}
13476	}
13477	break;
13478	}
13479
13480	default:
13481	{
13482	pVCpu->hm.s.u32HMError = uExitIntInfo;
13483	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13484	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13485	break;
13486	}
13487	}
13488	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13489	return rcStrict;
13490	}
13491
13492
13493	/**
13494	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13495	*/
13496	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13497	{
13498	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13499
13500	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13501	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13502	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13503	AssertRCReturn(rc, rc);
13504
13505	/* Evaluate and deliver pending events and resume guest execution. */
13506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13507	return VINF_SUCCESS;
13508	}
13509
13510
13511	/**
13512	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13513	*/
13514	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13515	{
13516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13517
13518	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13519	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13520	{
13521	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13522	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13523	}
13524
13525	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13526
13527	/*
13528	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13529	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13530	*/
13531	uint32_t fIntrState;
13532	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13533	AssertRCReturn(rc, rc);
13534	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13535	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13536	{
13537	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13538	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13539
13540	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13541	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13542	AssertRCReturn(rc, rc);
13543	}
13544
13545	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13546	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13547	AssertRCReturn(rc, rc);
13548
13549	/* Evaluate and deliver pending events and resume guest execution. */
13550	return VINF_SUCCESS;
13551	}
13552
13553
13554	/**
13555	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13556	*/
13557	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13558	{
13559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13560	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13561	}
13562
13563
13564	/**
13565	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13566	*/
13567	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13568	{
13569	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13570	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13571	}
13572
13573
13574	/**
13575	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13576	*/
13577	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13578	{
13579	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13580
13581	/*
13582	* Get the state we need and update the exit history entry.
13583	*/
13584	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13585	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13586	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13587	AssertRCReturn(rc, rc);
13588
13589	VBOXSTRICTRC rcStrict;
13590	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13591	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13592	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13593	if (!pExitRec)
13594	{
13595	/*
13596	* Regular CPUID instruction execution.
13597	*/
13598	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13599	if (rcStrict == VINF_SUCCESS)
13600	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13601	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13602	{
13603	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13604	rcStrict = VINF_SUCCESS;
13605	}
13606	}
13607	else
13608	{
13609	/*
13610	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13611	*/
13612	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13613	AssertRCReturn(rc2, rc2);
13614
13615	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13616	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13617
13618	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13619	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13620
13621	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13622	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13623	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13624	}
13625	return rcStrict;
13626	}
13627
13628
13629	/**
13630	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13631	*/
13632	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13633	{
13634	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13635
13636	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13637	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13638	AssertRCReturn(rc, rc);
13639
13640	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13641	return VINF_EM_RAW_EMULATE_INSTR;
13642
13643	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13644	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13645	}
13646
13647
13648	/**
13649	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13650	*/
13651	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13652	{
13653	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13654
13655	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13656	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13657	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13658	AssertRCReturn(rc, rc);
13659
13660	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13661	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13662	{
13663	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13664	we must reset offsetting on VM-entry. See @bugref{6634}. */
13665	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13666	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13668	}
13669	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13670	{
13671	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13672	rcStrict = VINF_SUCCESS;
13673	}
13674	return rcStrict;
13675	}
13676
13677
13678	/**
13679	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13680	*/
13681	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13682	{
13683	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13684
13685	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13686	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13687	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13688	AssertRCReturn(rc, rc);
13689
13690	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13691	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13692	{
13693	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13694	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13695	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13696	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13697	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13698	}
13699	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13700	{
13701	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13702	rcStrict = VINF_SUCCESS;
13703	}
13704	return rcStrict;
13705	}
13706
13707
13708	/**
13709	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13710	*/
13711	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13712	{
13713	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13714
13715	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13716	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13717	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13718	AssertRCReturn(rc, rc);
13719
13720	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13721	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13722	if (RT_LIKELY(rc == VINF_SUCCESS))
13723	{
13724	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13725	Assert(pVmxTransient->cbInstr == 2);
13726	}
13727	else
13728	{
13729	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13730	rc = VERR_EM_INTERPRETER;
13731	}
13732	return rc;
13733	}
13734
13735
13736	/**
13737	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13738	*/
13739	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13740	{
13741	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13742
13743	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13744	if (EMAreHypercallInstructionsEnabled(pVCpu))
13745	{
13746	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13747	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13748	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13749	AssertRCReturn(rc, rc);
13750
13751	/* Perform the hypercall. */
13752	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13753	if (rcStrict == VINF_SUCCESS)
13754	{
13755	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13756	AssertRCReturn(rc, rc);
13757	}
13758	else
13759	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13760	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13761	\|\| RT_FAILURE(rcStrict));
13762
13763	/* If the hypercall changes anything other than guest's general-purpose registers,
13764	we would need to reload the guest changed bits here before VM-entry. */
13765	}
13766	else
13767	Log4Func(("Hypercalls not enabled\n"));
13768
13769	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13770	if (RT_FAILURE(rcStrict))
13771	{
13772	hmR0VmxSetPendingXcptUD(pVCpu);
13773	rcStrict = VINF_SUCCESS;
13774	}
13775
13776	return rcStrict;
13777	}
13778
13779
13780	/**
13781	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13782	*/
13783	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13784	{
13785	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13786	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13787
13788	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13789	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13790	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13791	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13792	AssertRCReturn(rc, rc);
13793
13794	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13795
13796	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13797	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13798	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13799	{
13800	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13801	rcStrict = VINF_SUCCESS;
13802	}
13803	else
13804	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
13805	VBOXSTRICTRC_VAL(rcStrict)));
13806	return rcStrict;
13807	}
13808
13809
13810	/**
13811	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13812	*/
13813	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13814	{
13815	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13816
13817	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13818	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
13819	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13820	AssertRCReturn(rc, rc);
13821
13822	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
13823	if (rcStrict == VINF_SUCCESS)
13824	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13825	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13826	{
13827	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13828	rcStrict = VINF_SUCCESS;
13829	}
13830
13831	return rcStrict;
13832	}
13833
13834
13835	/**
13836	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13837	*/
13838	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13839	{
13840	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13841
13842	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13843	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13844	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13845	AssertRCReturn(rc, rc);
13846
13847	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
13848	if (RT_SUCCESS(rcStrict))
13849	{
13850	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13851	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
13852	rcStrict = VINF_SUCCESS;
13853	}
13854
13855	return rcStrict;
13856	}
13857
13858
13859	/**
13860	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13861	* VM-exit.
13862	*/
13863	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13864	{
13865	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13866	return VINF_EM_RESET;
13867	}
13868
13869
13870	/**
13871	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13872	*/
13873	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13874	{
13875	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13876
13877	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13878	AssertRCReturn(rc, rc);
13879
13880	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13881	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13882	rc = VINF_SUCCESS;
13883	else
13884	rc = VINF_EM_HALT;
13885
13886	if (rc != VINF_SUCCESS)
13887	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13888	return rc;
13889	}
13890
13891
13892	/**
13893	* VM-exit handler for instructions that result in a \#UD exception delivered to
13894	* the guest.
13895	*/
13896	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13897	{
13898	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13899	hmR0VmxSetPendingXcptUD(pVCpu);
13900	return VINF_SUCCESS;
13901	}
13902
13903
13904	/**
13905	* VM-exit handler for expiry of the VMX-preemption timer.
13906	*/
13907	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13908	{
13909	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13910
13911	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13912	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13913
13914	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13915	PVM pVM = pVCpu->CTX_SUFF(pVM);
13916	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13917	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13918	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13919	}
13920
13921
13922	/**
13923	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13924	*/
13925	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13926	{
13927	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13928
13929	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13930	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13931	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13932	AssertRCReturn(rc, rc);
13933
13934	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13935	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13936	: HM_CHANGED_RAISED_XCPT_MASK);
13937
13938	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13939	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13940
13941	return rcStrict;
13942	}
13943
13944
13945	/**
13946	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13947	*/
13948	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13949	{
13950	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13951	/** @todo Use VM-exit instruction information. */
13952	return VERR_EM_INTERPRETER;
13953	}
13954
13955
13956	/**
13957	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13958	* Error VM-exit.
13959	*/
13960	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13961	{
13962	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13963	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13964	AssertRCReturn(rc, rc);
13965
13966	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13967	if (RT_FAILURE(rc))
13968	return rc;
13969
13970	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13971	NOREF(uInvalidReason);
13972
13973	#ifdef VBOX_STRICT
13974	uint32_t fIntrState;
13975	RTHCUINTREG uHCReg;
13976	uint64_t u64Val;
13977	uint32_t u32Val;
13978	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13979	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13980	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13981	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13982	AssertRCReturn(rc, rc);
13983
13984	Log4(("uInvalidReason %u\n", uInvalidReason));
13985	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13986	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13987	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13988	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13989
13990	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13991	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13992	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13993	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13994	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13995	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13996	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13997	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13998	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13999	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14000	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14001	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14002
14003	hmR0DumpRegs(pVCpu);
14004	#endif
14005
14006	return VERR_VMX_INVALID_GUEST_STATE;
14007	}
14008
14009	/**
14010	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14011	*/
14012	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14013	{
14014	/*
14015	* Cummulative notes of all recognized but unexpected VM-exits.
14016	* This does -not- cover those VM-exits like a page-fault occurring when say nested-paging
14017	* is used.
14018	*
14019	* VMX_EXIT_INIT_SIGNAL:
14020	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14021	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14022	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14023	*
14024	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14025	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14026	* See Intel spec. "23.8 Restrictions on VMX operation".
14027	*
14028	* VMX_EXIT_SIPI:
14029	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14030	* activity state is used. We don't make use of it as our guests don't have direct
14031	* access to the host local APIC.
14032	*
14033	* See Intel spec. 25.3 "Other Causes of VM-exits".
14034	*
14035	* VMX_EXIT_IO_SMI:
14036	* VMX_EXIT_SMI:
14037	* This can only happen if we support dual-monitor treatment of SMI, which can be
14038	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14039	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14040	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14041	*
14042	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14043	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14044	*
14045	* VMX_EXIT_ERR_MACHINE_CHECK:
14046	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14047	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14048	* #MC exception abort class exception is raised. We thus cannot assume a
14049	* reasonable chance of continuing any sort of execution and we bail.
14050	*
14051	* See Intel spec. 15.1 "Machine-check Architecture".
14052	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14053	*
14054	* VMX_EXIT_ERR_MSR_LOAD:
14055	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14056	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14057	* execution.
14058	*
14059	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14060	*
14061	* VMX_EXIT_PML_FULL:
14062	* VMX_EXIT_VIRTUALIZED_EOI:
14063	* VMX_EXIT_APIC_WRITE:
14064	* We do not currently support any of these features and thus they are all unexpected
14065	* VM-exits.
14066	*/
14067	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14068	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14069	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14070	}
14071
14072
14073	/**
14074	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
14075	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
14076	* Conditional VM-exit.
14077	*/
14078	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14079	{
14080	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14081
14082	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
14083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
14084	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14085	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
14086	return VERR_EM_INTERPRETER;
14087	AssertMsgFailed(("Unexpected XDTR access\n"));
14088	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14089	}
14090
14091
14092	/**
14093	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
14094	*/
14095	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14096	{
14097	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14098
14099	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
14100	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14101	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
14102	return VERR_EM_INTERPRETER;
14103	AssertMsgFailed(("Unexpected RDRAND exit\n"));
14104	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14105	}
14106
14107
14108	/**
14109	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14110	*/
14111	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14112	{
14113	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14114
14115	/** @todo Optimize this: We currently drag in in the whole MSR state
14116	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14117	* MSRs required. That would require changes to IEM and possibly CPUM too.
14118	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14119	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14120	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14121	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14122	switch (idMsr)
14123	{
14124	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14125	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14126	}
14127
14128	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14129	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14130	AssertRCReturn(rc, rc);
14131
14132	Log4Func(("ecx=%#RX32\n", idMsr));
14133
14134	#ifdef VBOX_STRICT
14135	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14136	{
14137	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14138	&& idMsr != MSR_K6_EFER)
14139	{
14140	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14141	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14142	}
14143	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14144	{
14145	Assert(pVmcsInfo->pvMsrBitmap);
14146	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14147	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14148	{
14149	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14150	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14151	}
14152	}
14153	}
14154	#endif
14155
14156	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14158	if (rcStrict == VINF_SUCCESS)
14159	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14160	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14161	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14162	{
14163	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14164	rcStrict = VINF_SUCCESS;
14165	}
14166	else
14167	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14168
14169	return rcStrict;
14170	}
14171
14172
14173	/**
14174	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14175	*/
14176	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14177	{
14178	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14179
14180	/** @todo Optimize this: We currently drag in in the whole MSR state
14181	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14182	* MSRs required. That would require changes to IEM and possibly CPUM too.
14183	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14184	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14185	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14186
14187	/*
14188	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14189	* Although we don't need to fetch the base as it will be overwritten shortly, while
14190	* loading guest-state we would also load the entire segment register including limit
14191	* and attributes and thus we need to load them here.
14192	*/
14193	switch (idMsr)
14194	{
14195	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14196	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14197	}
14198
14199	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14200	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14201	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14202	AssertRCReturn(rc, rc);
14203
14204	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14205
14206	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14207	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14208
14209	if (rcStrict == VINF_SUCCESS)
14210	{
14211	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14212
14213	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14214	if ( idMsr == MSR_IA32_APICBASE
14215	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14216	&& idMsr <= MSR_IA32_X2APIC_END))
14217	{
14218	/*
14219	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14220	* When full APIC register virtualization is implemented we'll have to make
14221	* sure APIC state is saved from the VMCS before IEM changes it.
14222	*/
14223	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14224	}
14225	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14226	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14227	else if (idMsr == MSR_K6_EFER)
14228	{
14229	/*
14230	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14231	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14232	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14233	*/
14234	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14235	}
14236
14237	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14238	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14239	{
14240	switch (idMsr)
14241	{
14242	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14243	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14244	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14245	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14246	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14247	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14248	default:
14249	{
14250	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14251	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14252	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14253	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14254	break;
14255	}
14256	}
14257	}
14258	#ifdef VBOX_STRICT
14259	else
14260	{
14261	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14262	switch (idMsr)
14263	{
14264	case MSR_IA32_SYSENTER_CS:
14265	case MSR_IA32_SYSENTER_EIP:
14266	case MSR_IA32_SYSENTER_ESP:
14267	case MSR_K8_FS_BASE:
14268	case MSR_K8_GS_BASE:
14269	{
14270	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14271	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14272	}
14273
14274	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14275	default:
14276	{
14277	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14278	{
14279	/* EFER MSR writes are always intercepted. */
14280	if (idMsr != MSR_K6_EFER)
14281	{
14282	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14283	idMsr));
14284	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14285	}
14286	}
14287
14288	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14289	{
14290	Assert(pVmcsInfo->pvMsrBitmap);
14291	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14292	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14293	{
14294	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14295	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14296	}
14297	}
14298	break;
14299	}
14300	}
14301	}
14302	#endif /* VBOX_STRICT */
14303	}
14304	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14305	{
14306	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14307	rcStrict = VINF_SUCCESS;
14308	}
14309	else
14310	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14311
14312	return rcStrict;
14313	}
14314
14315
14316	/**
14317	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14318	*/
14319	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14320	{
14321	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14322
14323	/** @todo The guest has likely hit a contended spinlock. We might want to
14324	* poke a schedule different guest VCPU. */
14325	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14326	if (RT_SUCCESS(rc))
14327	return VINF_EM_RAW_INTERRUPT;
14328
14329	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14330	return rc;
14331	}
14332
14333
14334	/**
14335	* VM-exit handler for when the TPR value is lowered below the specified
14336	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14337	*/
14338	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14339	{
14340	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14341	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14342
14343	/*
14344	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14345	* We'll re-evaluate pending interrupts and inject them before the next VM
14346	* entry so we can just continue execution here.
14347	*/
14348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14349	return VINF_SUCCESS;
14350	}
14351
14352
14353	/**
14354	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14355	* VM-exit.
14356	*
14357	* @retval VINF_SUCCESS when guest execution can continue.
14358	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14359	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
14360	* interpreter.
14361	*/
14362	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14363	{
14364	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14365	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14366
14367	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14368	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14369	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14370	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14371	AssertRCReturn(rc, rc);
14372
14373	VBOXSTRICTRC rcStrict;
14374	PVM pVM = pVCpu->CTX_SUFF(pVM);
14375	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
14376	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14377	switch (uAccessType)
14378	{
14379	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14380	{
14381	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14382	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14383	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
14384	AssertMsg( rcStrict == VINF_SUCCESS
14385	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14386	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14387
14388	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14389	{
14390	case 0:
14391	{
14392	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14393	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14395	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14396
14397	/*
14398	* This is a kludge for handling switches back to real mode when we try to use
14399	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14400	* deal with special selector values, so we have to return to ring-3 and run
14401	* there till the selector values are V86 mode compatible.
14402	*
14403	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14404	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
14405	* at the end of this function.
14406	*/
14407	if ( rc == VINF_SUCCESS
14408	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
14409	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14410	&& (uOldCr0 & X86_CR0_PE)
14411	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
14412	{
14413	/** @todo check selectors rather than returning all the time. */
14414	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14415	rcStrict = VINF_EM_RESCHEDULE_REM;
14416	}
14417	break;
14418	}
14419
14420	case 2:
14421	{
14422	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14423	/* Nothing to do here, CR2 it's not part of the VMCS. */
14424	break;
14425	}
14426
14427	case 3:
14428	{
14429	Assert( !pVM->hm.s.fNestedPaging
14430	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14431	\|\| pVCpu->hm.s.fUsingDebugLoop);
14432	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14433	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14434	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14435	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14436	break;
14437	}
14438
14439	case 4:
14440	{
14441	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14442	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14443	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14444	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14445	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14446	break;
14447	}
14448
14449	case 8:
14450	{
14451	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14452	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14453	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14454	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14455	break;
14456	}
14457	default:
14458	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
14459	break;
14460	}
14461	break;
14462	}
14463
14464	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14465	{
14466	Assert( !pVM->hm.s.fNestedPaging
14467	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14468	\|\| pVCpu->hm.s.fUsingDebugLoop
14469	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
14470	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14471	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
14472	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14473
14474	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
14475	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
14476	AssertMsg( rcStrict == VINF_SUCCESS
14477	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14478	#ifdef VBOX_WITH_STATISTICS
14479	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14480	{
14481	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14482	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14483	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14484	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14485	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14486	}
14487	#endif
14488	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14489	VBOXSTRICTRC_VAL(rcStrict)));
14490	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14491	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14492	else
14493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14494	break;
14495	}
14496
14497	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14498	{
14499	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14500	AssertMsg( rcStrict == VINF_SUCCESS
14501	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14502
14503	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14505	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14506	break;
14507	}
14508
14509	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14510	{
14511	/*
14512	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
14513	*/
14514	RTGCPTR GCPtrEffDst;
14515	uint8_t const cbInstr = pVmxTransient->cbInstr;
14516	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
14517	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
14518	if (fMemOperand)
14519	{
14520	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14521	AssertRCReturn(rc, rc);
14522	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
14523	}
14524	else
14525	GCPtrEffDst = NIL_RTGCPTR;
14526
14527	rcStrict = hmR0VmxExitLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14528	break;
14529	}
14530
14531	default:
14532	{
14533	pVCpu->hm.s.u32HMError = uAccessType;
14534	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14535	VERR_VMX_UNEXPECTED_EXCEPTION);
14536	}
14537	}
14538
14539	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14540	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14541	if (rcStrict == VINF_IEM_RAISED_XCPT)
14542	{
14543	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14544	rcStrict = VINF_SUCCESS;
14545	}
14546
14547	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14548	NOREF(pVM);
14549	return rcStrict;
14550	}
14551
14552
14553	/**
14554	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14555	* VM-exit.
14556	*/
14557	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14558	{
14559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14560	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14561
14562	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14563	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14564	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14565	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14566	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14567	\| CPUMCTX_EXTRN_EFER);
14568	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14569	AssertRCReturn(rc, rc);
14570
14571	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14572	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14573	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14574	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14575	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14576	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14577	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14578	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14579
14580	/*
14581	* Update exit history to see if this exit can be optimized.
14582	*/
14583	VBOXSTRICTRC rcStrict;
14584	PCEMEXITREC pExitRec = NULL;
14585	if ( !fGstStepping
14586	&& !fDbgStepping)
14587	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14588	!fIOString
14589	? !fIOWrite
14590	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14591	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14592	: !fIOWrite
14593	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14594	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14595	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14596	if (!pExitRec)
14597	{
14598	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14599	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14600
14601	uint32_t const cbValue = s_aIOSizes[uIOSize];
14602	uint32_t const cbInstr = pVmxTransient->cbInstr;
14603	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14604	PVM pVM = pVCpu->CTX_SUFF(pVM);
14605	if (fIOString)
14606	{
14607	/*
14608	* INS/OUTS - I/O String instruction.
14609	*
14610	* Use instruction-information if available, otherwise fall back on
14611	* interpreting the instruction.
14612	*/
14613	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14614	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14615	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14616	if (fInsOutsInfo)
14617	{
14618	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14619	AssertRCReturn(rc2, rc2);
14620	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14621	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14622	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14623	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14624	if (fIOWrite)
14625	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14626	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14627	else
14628	{
14629	/*
14630	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14631	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14632	* See Intel Instruction spec. for "INS".
14633	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14634	*/
14635	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14636	}
14637	}
14638	else
14639	rcStrict = IEMExecOne(pVCpu);
14640
14641	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14642	fUpdateRipAlready = true;
14643	}
14644	else
14645	{
14646	/*
14647	* IN/OUT - I/O instruction.
14648	*/
14649	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14650	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14651	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14652	if (fIOWrite)
14653	{
14654	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14656	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14657	&& !pCtx->eflags.Bits.u1TF)
14658	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14659	}
14660	else
14661	{
14662	uint32_t u32Result = 0;
14663	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14664	if (IOM_SUCCESS(rcStrict))
14665	{
14666	/* Save result of I/O IN instr. in AL/AX/EAX. */
14667	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14668	}
14669	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14670	&& !pCtx->eflags.Bits.u1TF)
14671	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14672	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14673	}
14674	}
14675
14676	if (IOM_SUCCESS(rcStrict))
14677	{
14678	if (!fUpdateRipAlready)
14679	{
14680	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14681	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14682	}
14683
14684	/*
14685	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14686	* while booting Fedora 17 64-bit guest.
14687	*
14688	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14689	*/
14690	if (fIOString)
14691	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14692
14693	/*
14694	* If any I/O breakpoints are armed, we need to check if one triggered
14695	* and take appropriate action.
14696	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14697	*/
14698	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14699	AssertRCReturn(rc, rc);
14700
14701	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14702	* execution engines about whether hyper BPs and such are pending. */
14703	uint32_t const uDr7 = pCtx->dr[7];
14704	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14705	&& X86_DR7_ANY_RW_IO(uDr7)
14706	&& (pCtx->cr4 & X86_CR4_DE))
14707	\|\| DBGFBpIsHwIoArmed(pVM)))
14708	{
14709	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14710
14711	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14712	VMMRZCallRing3Disable(pVCpu);
14713	HM_DISABLE_PREEMPT(pVCpu);
14714
14715	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14716
14717	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14718	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14719	{
14720	/* Raise #DB. */
14721	if (fIsGuestDbgActive)
14722	ASMSetDR6(pCtx->dr[6]);
14723	if (pCtx->dr[7] != uDr7)
14724	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14725
14726	hmR0VmxSetPendingXcptDB(pVCpu);
14727	}
14728	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14729	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14730	else if ( rcStrict2 != VINF_SUCCESS
14731	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14732	rcStrict = rcStrict2;
14733	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14734
14735	HM_RESTORE_PREEMPT();
14736	VMMRZCallRing3Enable(pVCpu);
14737	}
14738	}
14739
14740	#ifdef VBOX_STRICT
14741	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14742	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14743	Assert(!fIOWrite);
14744	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14745	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14746	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14747	Assert(fIOWrite);
14748	else
14749	{
14750	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14751	* statuses, that the VMM device and some others may return. See
14752	* IOM_SUCCESS() for guidance. */
14753	AssertMsg( RT_FAILURE(rcStrict)
14754	\|\| rcStrict == VINF_SUCCESS
14755	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14756	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14757	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14758	\|\| rcStrict == VINF_EM_RAW_TO_R3
14759	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14760	# endif
14761	}
14762	#endif
14763	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14764	}
14765	else
14766	{
14767	/*
14768	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14769	*/
14770	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14771	AssertRCReturn(rc2, rc2);
14772	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14773	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14774	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14775	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14776	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14777	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14778
14779	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14780	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14781
14782	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14783	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14784	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14785	}
14786	return rcStrict;
14787	}
14788
14789
14790	/**
14791	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14792	* VM-exit.
14793	*/
14794	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14795	{
14796	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14797
14798	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14799	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14800	AssertRCReturn(rc, rc);
14801	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14802	{
14803	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14804	AssertRCReturn(rc, rc);
14805	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14806	{
14807	uint32_t uErrCode;
14808	RTGCUINTPTR GCPtrFaultAddress;
14809	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14810	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14811	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14812	if (fErrorCodeValid)
14813	{
14814	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14815	AssertRCReturn(rc, rc);
14816	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14817	}
14818	else
14819	uErrCode = 0;
14820
14821	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14822	&& uVector == X86_XCPT_PF)
14823	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14824	else
14825	GCPtrFaultAddress = 0;
14826
14827	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14828	AssertRCReturn(rc, rc);
14829
14830	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14831	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14832
14833	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14834	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14835	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14836	}
14837	}
14838
14839	/* Fall back to the interpreter to emulate the task-switch. */
14840	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14841	return VERR_EM_INTERPRETER;
14842	}
14843
14844
14845	/**
14846	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14847	*/
14848	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14849	{
14850	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14851
14852	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14853	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14854	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14855	AssertRCReturn(rc, rc);
14856	return VINF_EM_DBG_STEPPED;
14857	}
14858
14859
14860	/**
14861	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14862	*/
14863	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14864	{
14865	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14866	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14867
14868	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14869	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14870	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14871	{
14872	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14873	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14874	{
14875	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14876	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14877	}
14878	}
14879	else
14880	{
14881	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14882	rcStrict1 = VINF_SUCCESS;
14883	return rcStrict1;
14884	}
14885
14886	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14887	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14888	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14889	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14890	AssertRCReturn(rc, rc);
14891
14892	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14893	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14894	VBOXSTRICTRC rcStrict2;
14895	switch (uAccessType)
14896	{
14897	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14898	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14899	{
14900	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14901	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14902	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14903
14904	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14905	GCPhys &= PAGE_BASE_GC_MASK;
14906	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14907	PVM pVM = pVCpu->CTX_SUFF(pVM);
14908	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14909	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14910
14911	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14912	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14913	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14914	CPUMCTX2CORE(pCtx), GCPhys);
14915	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14916	if ( rcStrict2 == VINF_SUCCESS
14917	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14918	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14919	{
14920	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14921	\| HM_CHANGED_GUEST_APIC_TPR);
14922	rcStrict2 = VINF_SUCCESS;
14923	}
14924	break;
14925	}
14926
14927	default:
14928	Log4Func(("uAccessType=%#x\n", uAccessType));
14929	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14930	break;
14931	}
14932
14933	if (rcStrict2 != VINF_SUCCESS)
14934	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14935	return rcStrict2;
14936	}
14937
14938
14939	/**
14940	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14941	* VM-exit.
14942	*/
14943	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14944	{
14945	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14946
14947	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14948	if (pVmxTransient->fWasGuestDebugStateActive)
14949	{
14950	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14951	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14952	}
14953
14954	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14955	if ( !pVCpu->hm.s.fSingleInstruction
14956	&& !pVmxTransient->fWasHyperDebugStateActive)
14957	{
14958	Assert(!DBGFIsStepping(pVCpu));
14959	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14960
14961	/* Don't intercept MOV DRx any more. */
14962	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14963	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14964	AssertRCReturn(rc, rc);
14965
14966	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14967	VMMRZCallRing3Disable(pVCpu);
14968	HM_DISABLE_PREEMPT(pVCpu);
14969
14970	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14971	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14972	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14973
14974	HM_RESTORE_PREEMPT();
14975	VMMRZCallRing3Enable(pVCpu);
14976
14977	#ifdef VBOX_WITH_STATISTICS
14978	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14979	AssertRCReturn(rc, rc);
14980	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14981	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14982	else
14983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14984	#endif
14985	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14986	return VINF_SUCCESS;
14987	}
14988
14989	/*
14990	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14991	* The EFER MSR is always up-to-date.
14992	* Update the segment registers and DR7 from the CPU.
14993	*/
14994	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14995	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14996	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14997	AssertRCReturn(rc, rc);
14998	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14999
15000	PVM pVM = pVCpu->CTX_SUFF(pVM);
15001	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15002	{
15003	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15004	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15005	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15006	if (RT_SUCCESS(rc))
15007	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15008	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15009	}
15010	else
15011	{
15012	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15013	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15014	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15015	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15016	}
15017
15018	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15019	if (RT_SUCCESS(rc))
15020	{
15021	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15022	AssertRCReturn(rc2, rc2);
15023	return VINF_SUCCESS;
15024	}
15025	return rc;
15026	}
15027
15028
15029	/**
15030	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15031	* Conditional VM-exit.
15032	*/
15033	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15034	{
15035	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15036	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15037
15038	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15039	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15040	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15041	{
15042	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
15043	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
15044	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15045	{
15046	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
15047	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15048	}
15049	}
15050	else
15051	{
15052	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15053	rcStrict1 = VINF_SUCCESS;
15054	return rcStrict1;
15055	}
15056
15057	/*
15058	* Get sufficent state and update the exit history entry.
15059	*/
15060	RTGCPHYS GCPhys;
15061	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15062	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15063	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15064	AssertRCReturn(rc, rc);
15065
15066	VBOXSTRICTRC rcStrict;
15067	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15068	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15069	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15070	if (!pExitRec)
15071	{
15072	/*
15073	* If we succeed, resume guest execution.
15074	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15075	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15076	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15077	* weird case. See @bugref{6043}.
15078	*/
15079	PVM pVM = pVCpu->CTX_SUFF(pVM);
15080	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15081	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15082	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15083	if ( rcStrict == VINF_SUCCESS
15084	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15085	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15086	{
15087	/* Successfully handled MMIO operation. */
15088	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15089	\| HM_CHANGED_GUEST_APIC_TPR);
15090	rcStrict = VINF_SUCCESS;
15091	}
15092	}
15093	else
15094	{
15095	/*
15096	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15097	*/
15098	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15099	AssertRCReturn(rc2, rc2);
15100
15101	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15102	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15103
15104	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15105	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15106
15107	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15108	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15109	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15110	}
15111	return VBOXSTRICTRC_TODO(rcStrict);
15112	}
15113
15114
15115	/**
15116	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15117	* VM-exit.
15118	*/
15119	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15120	{
15121	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15122	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15123
15124	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15125	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15126	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15127	{
15128	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
15129	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15130	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
15131	}
15132	else
15133	{
15134	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15135	rcStrict1 = VINF_SUCCESS;
15136	return rcStrict1;
15137	}
15138
15139	RTGCPHYS GCPhys;
15140	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15141	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15142	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15143	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15144	AssertRCReturn(rc, rc);
15145
15146	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
15147	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
15148
15149	RTGCUINT uErrorCode = 0;
15150	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15151	uErrorCode \|= X86_TRAP_PF_ID;
15152	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15153	uErrorCode \|= X86_TRAP_PF_RW;
15154	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15155	uErrorCode \|= X86_TRAP_PF_P;
15156
15157	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15158
15159
15160	/* Handle the pagefault trap for the nested shadow table. */
15161	PVM pVM = pVCpu->CTX_SUFF(pVM);
15162	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15163
15164	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
15165	pCtx->cs.Sel, pCtx->rip));
15166
15167	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15168	TRPMResetTrap(pVCpu);
15169
15170	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15171	if ( rcStrict2 == VINF_SUCCESS
15172	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15173	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
15174	{
15175	/* Successfully synced our nested page tables. */
15176	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15177	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15178	return VINF_SUCCESS;
15179	}
15180
15181	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15182	return rcStrict2;
15183	}
15184
15185	/** @} */
15186
15187	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15188	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
15189	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15190
15191	/**
15192	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
15193	*/
15194	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15195	{
15196	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15198
15199	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15200	AssertRCReturn(rc, rc);
15201
15202	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15203	{
15204	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15205	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15206
15207	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15208	* provides VM-exit instruction length. If this causes problem later,
15209	* disassemble the instruction like it's done on AMD-V. */
15210	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15211	AssertRCReturn(rc2, rc2);
15212	return rc;
15213	}
15214
15215	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15216	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15217	return rc;
15218	}
15219
15220
15221	/**
15222	* VM-exit exception handler for \#BP (Breakpoint exception).
15223	*/
15224	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15225	{
15226	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15228
15229	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15230	AssertRCReturn(rc, rc);
15231
15232	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15233	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15234	if (rc == VINF_EM_RAW_GUEST_TRAP)
15235	{
15236	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15237	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15238	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15239	AssertRCReturn(rc, rc);
15240
15241	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15242	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15243	}
15244
15245	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15246	return rc;
15247	}
15248
15249
15250	/**
15251	* VM-exit exception handler for \#AC (alignment check exception).
15252	*/
15253	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15254	{
15255	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15256
15257	/*
15258	* Re-inject it. We'll detect any nesting before getting here.
15259	*/
15260	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15261	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15262	AssertRCReturn(rc, rc);
15263	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15264
15265	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15266	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15267	return VINF_SUCCESS;
15268	}
15269
15270
15271	/**
15272	* VM-exit exception handler for \#DB (Debug exception).
15273	*/
15274	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15275	{
15276	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15278
15279	/*
15280	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
15281	* for processing.
15282	*/
15283	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15284
15285	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15286	uint64_t const uDR6 = X86_DR6_INIT_VAL
15287	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15288	\| X86_DR6_BD \| X86_DR6_BS));
15289
15290	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15291	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15292	Log6Func(("rc=%Rrc\n", rc));
15293	if (rc == VINF_EM_RAW_GUEST_TRAP)
15294	{
15295	/*
15296	* The exception was for the guest. Update DR6, DR7.GD and
15297	* IA32_DEBUGCTL.LBR before forwarding it.
15298	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15299	*/
15300	VMMRZCallRing3Disable(pVCpu);
15301	HM_DISABLE_PREEMPT(pVCpu);
15302
15303	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15304	pCtx->dr[6] \|= uDR6;
15305	if (CPUMIsGuestDebugStateActive(pVCpu))
15306	ASMSetDR6(pCtx->dr[6]);
15307
15308	HM_RESTORE_PREEMPT();
15309	VMMRZCallRing3Enable(pVCpu);
15310
15311	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15312	AssertRCReturn(rc, rc);
15313
15314	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15315	pCtx->dr[7] &= ~X86_DR7_GD;
15316
15317	/* Paranoia. */
15318	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15319	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15320
15321	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15322	AssertRCReturn(rc, rc);
15323
15324	/*
15325	* Raise #DB in the guest.
15326	*
15327	* It is important to reflect exactly what the VM-exit gave us (preserving the
15328	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15329	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15330	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15331	*
15332	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15333	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15334	*/
15335	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15336	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15337	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15338	AssertRCReturn(rc, rc);
15339	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15340	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15341	return VINF_SUCCESS;
15342	}
15343
15344	/*
15345	* Not a guest trap, must be a hypervisor related debug event then.
15346	* Update DR6 in case someone is interested in it.
15347	*/
15348	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15349	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15350	CPUMSetHyperDR6(pVCpu, uDR6);
15351
15352	return rc;
15353	}
15354
15355
15356	/**
15357	* Hacks its way around the lovely mesa driver's backdoor accesses.
15358	*
15359	* @sa hmR0SvmHandleMesaDrvGp.
15360	*/
15361	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15362	{
15363	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15364	RT_NOREF(pCtx);
15365
15366	/* For now we'll just skip the instruction. */
15367	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15368	}
15369
15370
15371	/**
15372	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15373	* backdoor logging w/o checking what it is running inside.
15374	*
15375	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15376	* backdoor port and magic numbers loaded in registers.
15377	*
15378	* @returns true if it is, false if it isn't.
15379	* @sa hmR0SvmIsMesaDrvGp.
15380	*/
15381	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15382	{
15383	/* 0xed: IN eAX,dx */
15384	uint8_t abInstr[1];
15385	if (pVmxTransient->cbInstr != sizeof(abInstr))
15386	return false;
15387
15388	/* Check that it is #GP(0). */
15389	if (pVmxTransient->uExitIntErrorCode != 0)
15390	return false;
15391
15392	/* Check magic and port. */
15393	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15394	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15395	if (pCtx->rax != UINT32_C(0x564d5868))
15396	return false;
15397	if (pCtx->dx != UINT32_C(0x5658))
15398	return false;
15399
15400	/* Flat ring-3 CS. */
15401	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15402	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15403	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15404	if (pCtx->cs.Attr.n.u2Dpl != 3)
15405	return false;
15406	if (pCtx->cs.u64Base != 0)
15407	return false;
15408
15409	/* Check opcode. */
15410	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15411	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15412	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15413	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15414	if (RT_FAILURE(rc))
15415	return false;
15416	if (abInstr[0] != 0xed)
15417	return false;
15418
15419	return true;
15420	}
15421
15422
15423	/**
15424	* VM-exit exception handler for \#GP (General-protection exception).
15425	*
15426	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15427	*/
15428	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15429	{
15430	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15431	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15432
15433	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15434	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15435	if (pVmcsInfo->RealMode.fRealOnV86Active)
15436	{ /* likely */ }
15437	else
15438	{
15439	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15440	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15441	#endif
15442	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15443	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15444	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15445	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15446	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15447	AssertRCReturn(rc, rc);
15448	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15449	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15450
15451	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15452	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15453	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15454	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15455	else
15456	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15457	return rc;
15458	}
15459
15460	Assert(CPUMIsGuestInRealModeEx(pCtx));
15461	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15462
15463	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15464	AssertRCReturn(rc, rc);
15465
15466	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15467	if (rcStrict == VINF_SUCCESS)
15468	{
15469	if (!CPUMIsGuestInRealModeEx(pCtx))
15470	{
15471	/*
15472	* The guest is no longer in real-mode, check if we can continue executing the
15473	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15474	*/
15475	pVmcsInfo->RealMode.fRealOnV86Active = false;
15476	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15477	{
15478	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15479	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15480	}
15481	else
15482	{
15483	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15484	rcStrict = VINF_EM_RESCHEDULE;
15485	}
15486	}
15487	else
15488	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15489	}
15490	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15491	{
15492	rcStrict = VINF_SUCCESS;
15493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15494	}
15495	return VBOXSTRICTRC_VAL(rcStrict);
15496	}
15497
15498
15499	/**
15500	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15501	* the exception reported in the VMX transient structure back into the VM.
15502	*
15503	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15504	* up-to-date.
15505	*/
15506	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15507	{
15508	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15509	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15510	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15511	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15512	("uVector=%#x u32XcptBitmap=%#X32\n",
15513	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15514	NOREF(pVmcsInfo);
15515	#endif
15516
15517	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15518	hmR0VmxCheckExitDueToEventDelivery(). */
15519	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15520	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15521	AssertRCReturn(rc, rc);
15522	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15523
15524	#ifdef DEBUG_ramshankar
15525	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15526	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15527	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15528	#endif
15529
15530	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15531	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15532	return VINF_SUCCESS;
15533	}
15534
15535
15536	/**
15537	* VM-exit exception handler for \#PF (Page-fault exception).
15538	*/
15539	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15540	{
15541	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15542	PVM pVM = pVCpu->CTX_SUFF(pVM);
15543	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15544	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15545	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15546	AssertRCReturn(rc, rc);
15547
15548	if (!pVM->hm.s.fNestedPaging)
15549	{ /* likely */ }
15550	else
15551	{
15552	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15553	Assert(pVCpu->hm.s.fUsingDebugLoop);
15554	#endif
15555	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15556	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15557	{
15558	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15559	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15560	}
15561	else
15562	{
15563	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15564	hmR0VmxSetPendingXcptDF(pVCpu);
15565	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15566	}
15567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15568	return rc;
15569	}
15570
15571	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15572	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15573	if (pVmxTransient->fVectoringPF)
15574	{
15575	Assert(pVCpu->hm.s.Event.fPending);
15576	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15577	}
15578
15579	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15580	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15581	AssertRCReturn(rc, rc);
15582
15583	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15584	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15585
15586	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15587	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15588
15589	Log4Func(("#PF: rc=%Rrc\n", rc));
15590	if (rc == VINF_SUCCESS)
15591	{
15592	/*
15593	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15594	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15595	*/
15596	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15597	TRPMResetTrap(pVCpu);
15598	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15599	return rc;
15600	}
15601
15602	if (rc == VINF_EM_RAW_GUEST_TRAP)
15603	{
15604	if (!pVmxTransient->fVectoringDoublePF)
15605	{
15606	/* It's a guest page fault and needs to be reflected to the guest. */
15607	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15608	TRPMResetTrap(pVCpu);
15609	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15610	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15611	uGstErrorCode, pVmxTransient->uExitQual);
15612	}
15613	else
15614	{
15615	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15616	TRPMResetTrap(pVCpu);
15617	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15618	hmR0VmxSetPendingXcptDF(pVCpu);
15619	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15620	}
15621
15622	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15623	return VINF_SUCCESS;
15624	}
15625
15626	TRPMResetTrap(pVCpu);
15627	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15628	return rc;
15629	}
15630
15631
15632	/**
15633	* VM-exit exception handler for LMSW.
15634	*/
15635	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
15636	{
15637	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
15638	AssertMsg( rcStrict == VINF_SUCCESS
15639	\|\| rcStrict == VINF_IEM_RAISED_XCPT
15640	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15641	Log4Func(("rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
15642
15643	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15644	if (rcStrict == VINF_IEM_RAISED_XCPT)
15645	{
15646	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15647	rcStrict = VINF_SUCCESS;
15648	}
15649	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
15650	return rcStrict;
15651	}
15652
15653
15654	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15655	/** @name VMX instruction handlers.
15656	* @{
15657	*/
15658	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15659	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15660	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15661
15662	/**
15663	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15664	*/
15665	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15666	{
15667	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15668
15669	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15670	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15671	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15672	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15673	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15674	AssertRCReturn(rc, rc);
15675
15676	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15677
15678	VMXVEXITINFO ExitInfo;
15679	RT_ZERO(ExitInfo);
15680	ExitInfo.uReason = pVmxTransient->uExitReason;
15681	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15682	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15683	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15684	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15685
15686	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15687	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15688	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15689	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15690	{
15691	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15692	rcStrict = VINF_SUCCESS;
15693	}
15694	return rcStrict;
15695	}
15696
15697
15698	/**
15699	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15700	*/
15701	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15702	{
15703	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15704
15705	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15706	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15707	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15708	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15709	AssertRCReturn(rc, rc);
15710
15711	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15712
15713	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15714	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15715	{
15716	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15717	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15718	}
15719	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15720	return rcStrict;
15721	}
15722
15723
15724	/**
15725	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15726	*/
15727	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15728	{
15729	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15730
15731	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15732	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15733	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15734	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15735	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15736	AssertRCReturn(rc, rc);
15737
15738	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15739
15740	VMXVEXITINFO ExitInfo;
15741	RT_ZERO(ExitInfo);
15742	ExitInfo.uReason = pVmxTransient->uExitReason;
15743	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15744	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15745	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15746	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15747
15748	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15749	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15750	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15751	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15752	{
15753	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15754	rcStrict = VINF_SUCCESS;
15755	}
15756	return rcStrict;
15757	}
15758
15759
15760	/**
15761	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15762	*/
15763	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15764	{
15765	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15766
15767	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15768	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15769	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15770	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15771	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15772	AssertRCReturn(rc, rc);
15773
15774	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15775
15776	VMXVEXITINFO ExitInfo;
15777	RT_ZERO(ExitInfo);
15778	ExitInfo.uReason = pVmxTransient->uExitReason;
15779	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15780	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15781	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15782	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15783
15784	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15785	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15786	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15787	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15788	{
15789	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15790	rcStrict = VINF_SUCCESS;
15791	}
15792	return rcStrict;
15793	}
15794
15795
15796	/**
15797	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15798	*/
15799	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15800	{
15801	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15802
15803	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15804	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15805	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15806	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15807	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15808	AssertRCReturn(rc, rc);
15809
15810	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15811
15812	VMXVEXITINFO ExitInfo;
15813	RT_ZERO(ExitInfo);
15814	ExitInfo.uReason = pVmxTransient->uExitReason;
15815	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15816	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15817	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15818	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15819	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15820
15821	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15822	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15823	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15824	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15825	{
15826	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15827	rcStrict = VINF_SUCCESS;
15828	}
15829	return rcStrict;
15830	}
15831
15832
15833	/**
15834	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15835	*/
15836	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15837	{
15838	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15839
15840	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15841	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15842	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15843	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15844	AssertRCReturn(rc, rc);
15845
15846	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15847
15848	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15849	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15850	{
15851	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15852	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15853	}
15854	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15855	return rcStrict;
15856	}
15857
15858
15859	/**
15860	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15861	*/
15862	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15863	{
15864	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15865
15866	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15867	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15868	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15869	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15870	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15871	AssertRCReturn(rc, rc);
15872
15873	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15874
15875	VMXVEXITINFO ExitInfo;
15876	RT_ZERO(ExitInfo);
15877	ExitInfo.uReason = pVmxTransient->uExitReason;
15878	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15879	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15880	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15881	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15882	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15883
15884	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15885	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15886	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15887	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15888	{
15889	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15890	rcStrict = VINF_SUCCESS;
15891	}
15892	return rcStrict;
15893	}
15894
15895
15896	/**
15897	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15898	*/
15899	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15900	{
15901	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15902
15903	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15904	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15905	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15906	AssertRCReturn(rc, rc);
15907
15908	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15909
15910	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15911	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15912	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15913	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15914	{
15915	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15916	rcStrict = VINF_SUCCESS;
15917	}
15918	return rcStrict;
15919	}
15920
15921
15922	/**
15923	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15924	*/
15925	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15926	{
15927	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15928
15929	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15930	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15931	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15932	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15933	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15934	AssertRCReturn(rc, rc);
15935
15936	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15937
15938	VMXVEXITINFO ExitInfo;
15939	RT_ZERO(ExitInfo);
15940	ExitInfo.uReason = pVmxTransient->uExitReason;
15941	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15942	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15943	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15944	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15945
15946	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15947	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15948	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15949	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15950	{
15951	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15952	rcStrict = VINF_SUCCESS;
15953	}
15954	return rcStrict;
15955	}
15956
15957
15958	/**
15959	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15960	*/
15961	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15962	{
15963	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15964
15965	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15966	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15967	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15968	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15969	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15970	AssertRCReturn(rc, rc);
15971
15972	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15973
15974	VMXVEXITINFO ExitInfo;
15975	RT_ZERO(ExitInfo);
15976	ExitInfo.uReason = pVmxTransient->uExitReason;
15977	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15978	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15979	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15980	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15981
15982	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15983	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15984	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15985	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15986	{
15987	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15988	rcStrict = VINF_SUCCESS;
15989	}
15990	return rcStrict;
15991	}
15992
15993	/** @} */
15994	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15995

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78772

Download in other formats: