HMVMXR0.cpp@ 78777

Last change on this file since 78777 was 78777, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Mov-to-CR0/CR4 VM-exit handling.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 670.0 KB

Line
1	/* $Id: HMVMXR0.cpp 78777 2019-05-27 08:53:59Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
366	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369	static FNVMXEXITHANDLER hmR0VmxExitHlt;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
371	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
376	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
377	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
378	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
379	static FNVMXEXITHANDLER hmR0VmxExitVmread;
380	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
381	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
382	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
383	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
384	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
385	#endif
386	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
387	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
388	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
389	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
390	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
391	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
392	static FNVMXEXITHANDLER hmR0VmxExitMwait;
393	static FNVMXEXITHANDLER hmR0VmxExitMtf;
394	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395	static FNVMXEXITHANDLER hmR0VmxExitPause;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
404	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
410	/** @} */
411
412	/** @name Helpers for hardware exceptions VM-exit handlers.
413	* @{
414	*/
415	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
416	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
417	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
418	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
419	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
420	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
421	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
423	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, uint8_t cbInstr);
424	/** @} */
425
426
427	/*********************************************************************************************************************************
428	* Global Variables *
429	*********************************************************************************************************************************/
430	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
431	static const uint32_t g_aVmcsCacheSegBase[] =
432	{
433	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
434	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
435	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
436	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
437	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
439	};
440	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
441	#endif
442	static const uint32_t g_aVmcsSegBase[] =
443	{
444	VMX_VMCS_GUEST_ES_BASE,
445	VMX_VMCS_GUEST_CS_BASE,
446	VMX_VMCS_GUEST_SS_BASE,
447	VMX_VMCS_GUEST_DS_BASE,
448	VMX_VMCS_GUEST_FS_BASE,
449	VMX_VMCS_GUEST_GS_BASE
450	};
451	static const uint32_t g_aVmcsSegSel[] =
452	{
453	VMX_VMCS16_GUEST_ES_SEL,
454	VMX_VMCS16_GUEST_CS_SEL,
455	VMX_VMCS16_GUEST_SS_SEL,
456	VMX_VMCS16_GUEST_DS_SEL,
457	VMX_VMCS16_GUEST_FS_SEL,
458	VMX_VMCS16_GUEST_GS_SEL
459	};
460	static const uint32_t g_aVmcsSegLimit[] =
461	{
462	VMX_VMCS32_GUEST_ES_LIMIT,
463	VMX_VMCS32_GUEST_CS_LIMIT,
464	VMX_VMCS32_GUEST_SS_LIMIT,
465	VMX_VMCS32_GUEST_DS_LIMIT,
466	VMX_VMCS32_GUEST_FS_LIMIT,
467	VMX_VMCS32_GUEST_GS_LIMIT
468	};
469	static const uint32_t g_aVmcsSegAttr[] =
470	{
471	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
472	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
473	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
474	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
475	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
477	};
478	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
479	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
480	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
481	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
482
483	#ifdef HMVMX_USE_FUNCTION_TABLE
484	/**
485	* VMX_EXIT dispatch table.
486	*/
487	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
488	{
489	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
490	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
491	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
492	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
493	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
494	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
495	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
496	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
497	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
498	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
499	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
500	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
501	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
502	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
503	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
504	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
505	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
506	/* 17 VMX_EXIT_RSM */ hmR0VmxExitSetPendingXcptUD,
507	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
508	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
509	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
510	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
511	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
512	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
513	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
514	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
515	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
516	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
517	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
518	#else
519	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
520	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
521	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
522	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
523	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
524	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
525	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
526	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
527	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
528	#endif
529	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
530	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
531	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
532	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
533	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
534	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
535	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
536	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
537	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
538	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
539	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
540	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
541	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
542	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
543	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
544	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
545	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
546	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
547	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
548	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
549	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
550	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
551	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
552	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
553	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
554	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
555	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
556	#else
557	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
558	#endif
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitSetPendingXcptUD,
566	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitSetPendingXcptUD,
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
571	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
572	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitSetPendingXcptUD,
573	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitSetPendingXcptUD,
574	};
575	#endif /* HMVMX_USE_FUNCTION_TABLE */
576
577	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
578	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
579	{
580	/* 0 */ "(Not Used)",
581	/* 1 */ "VMCALL executed in VMX root operation.",
582	/* 2 */ "VMCLEAR with invalid physical address.",
583	/* 3 */ "VMCLEAR with VMXON pointer.",
584	/* 4 */ "VMLAUNCH with non-clear VMCS.",
585	/* 5 */ "VMRESUME with non-launched VMCS.",
586	/* 6 */ "VMRESUME after VMXOFF",
587	/* 7 */ "VM-entry with invalid control fields.",
588	/* 8 */ "VM-entry with invalid host state fields.",
589	/* 9 */ "VMPTRLD with invalid physical address.",
590	/* 10 */ "VMPTRLD with VMXON pointer.",
591	/* 11 */ "VMPTRLD with incorrect revision identifier.",
592	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
593	/* 13 */ "VMWRITE to read-only VMCS component.",
594	/* 14 */ "(Not Used)",
595	/* 15 */ "VMXON executed in VMX root operation.",
596	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
597	/* 17 */ "VM-entry with non-launched executing VMCS.",
598	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
599	/* 19 */ "VMCALL with non-clear VMCS.",
600	/* 20 */ "VMCALL with invalid VM-exit control fields.",
601	/* 21 */ "(Not Used)",
602	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
603	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
604	/* 24 */ "VMCALL with invalid SMM-monitor features.",
605	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
606	/* 26 */ "VM-entry with events blocked by MOV SS.",
607	/* 27 */ "(Not Used)",
608	/* 28 */ "Invalid operand to INVEPT/INVVPID."
609	};
610	#endif /* VBOX_STRICT */
611
612
613	/**
614	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
615	*
616	* Any bit set in this mask is owned by the host/hypervisor and would cause a
617	* VM-exit when modified by the guest.
618	*
619	* @returns The static CR0 guest/host mask.
620	* @param pVCpu The cross context virtual CPU structure.
621	*/
622	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
623	{
624	/*
625	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
626	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
627	*/
628	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
629	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
630	* and @bugref{6944}. */
631	PVM pVM = pVCpu->CTX_SUFF(pVM);
632	return ( X86_CR0_PE
633	\| X86_CR0_NE
634	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
635	\| X86_CR0_PG
636	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
637	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
638	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
639	}
640
641
642	/**
643	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
644	*
645	* Any bit set in this mask is owned by the host/hypervisor and would cause a
646	* VM-exit when modified by the guest.
647	*
648	* @returns The static CR4 guest/host mask.
649	* @param pVCpu The cross context virtual CPU structure.
650	*/
651	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
652	{
653	/*
654	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
655	* these bits are reserved on hardware that does not support them. Since the
656	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
657	* these bits and handle it depending on whether we expose them to the guest.
658	*/
659	PVM pVM = pVCpu->CTX_SUFF(pVM);
660	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
661	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
662	return ( X86_CR4_VMXE
663	\| X86_CR4_VME
664	\| X86_CR4_PAE
665	\| X86_CR4_PGE
666	\| X86_CR4_PSE
667	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
668	\| (fPcid ? X86_CR4_PCIDE : 0));
669	}
670
671
672	/**
673	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
674	* area.
675	*
676	* @returns @c true if it's different, @c false otherwise.
677	* @param pVmcsInfo The VMCS info. object.
678	*/
679	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
680	{
681	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
682	&& pVmcsInfo->pvGuestMsrStore);
683	}
684
685
686	/**
687	* Adds one or more exceptions to the exception bitmap and commits it to the current
688	* VMCS.
689	*
690	* @returns VBox status code.
691	* @param pVmxTransient The VMX-transient structure.
692	* @param uXcptMask The exception(s) to add.
693	*/
694	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
695	{
696	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
697	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
698	if ((uXcptBitmap & uXcptMask) != uXcptMask)
699	{
700	uXcptBitmap \|= uXcptMask;
701	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
702	AssertRCReturn(rc, rc);
703	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
704	}
705	return VINF_SUCCESS;
706	}
707
708
709	/**
710	* Adds an exception to the exception bitmap and commits it to the current VMCS.
711	*
712	* @returns VBox status code.
713	* @param pVmxTransient The VMX-transient structure.
714	* @param uXcpt The exception to add.
715	*/
716	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
717	{
718	Assert(uXcpt <= X86_XCPT_LAST);
719	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
720	}
721
722
723	/**
724	* Remove one or more exceptions from the exception bitmap and commits it to the
725	* current VMCS.
726	*
727	* This takes care of not removing the exception intercept if a nested-guest
728	* requires the exception to be intercepted.
729	*
730	* @returns VBox status code.
731	* @param pVCpu The cross context virtual CPU structure.
732	* @param pVmxTransient The VMX-transient structure.
733	* @param uXcptMask The exception(s) to remove.
734	*/
735	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
736	{
737	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
738	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
739	if (u32XcptBitmap & uXcptMask)
740	{
741	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
742	if (!pVmxTransient->fIsNestedGuest)
743	{ /* likely */ }
744	else
745	{
746	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
747	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
748	}
749	#endif
750	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
751	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
752	\| RT_BIT(X86_XCPT_DE)
753	\| RT_BIT(X86_XCPT_NM)
754	\| RT_BIT(X86_XCPT_TS)
755	\| RT_BIT(X86_XCPT_UD)
756	\| RT_BIT(X86_XCPT_NP)
757	\| RT_BIT(X86_XCPT_SS)
758	\| RT_BIT(X86_XCPT_GP)
759	\| RT_BIT(X86_XCPT_PF)
760	\| RT_BIT(X86_XCPT_MF));
761	#elif defined(HMVMX_ALWAYS_TRAP_PF)
762	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
763	#endif
764	if (uXcptMask)
765	{
766	/* Validate we are not removing any essential exception intercepts. */
767	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
768	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
769	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
770
771	/* Remove it from the exception bitmap. */
772	u32XcptBitmap &= ~uXcptMask;
773
774	/* Commit and update the cache if necessary. */
775	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
776	{
777	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
778	AssertRCReturn(rc, rc);
779	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
780	}
781	}
782	}
783	return VINF_SUCCESS;
784	}
785
786
787	/**
788	* Remove an exceptions from the exception bitmap and commits it to the current
789	* VMCS.
790	*
791	* @returns VBox status code.
792	* @param pVCpu The cross context virtual CPU structure.
793	* @param pVmxTransient The VMX-transient structure.
794	* @param uXcpt The exception to remove.
795	*/
796	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
797	{
798	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
799	}
800
801
802	/**
803	* Loads the VMCS specified by the VMCS info. object.
804	*
805	* @returns VBox status code.
806	* @param pVmcsInfo The VMCS info. object.
807	*/
808	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
809	{
810	Assert(pVmcsInfo);
811	Assert(pVmcsInfo->HCPhysVmcs);
812	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
813
814	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
815	{
816	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
817	if (RT_SUCCESS(rc))
818	{
819	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
820	return VINF_SUCCESS;
821	}
822	return rc;
823	}
824	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
825	}
826
827
828	/**
829	* Clears the VMCS specified by the VMCS info. object.
830	*
831	* @returns VBox status code.
832	* @param pVmcsInfo The VMCS info. object.
833	*/
834	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
835	{
836	Assert(pVmcsInfo);
837	Assert(pVmcsInfo->HCPhysVmcs);
838	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
839
840	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
841	if (RT_SUCCESS(rc))
842	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
843	return rc;
844	}
845
846
847	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
848	/**
849	* Switches the current VMCS to the one specified.
850	*
851	* @returns VBox status code.
852	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
853	* @param pVmcsInfoTo The VMCS info. object we are switching to.
854	*
855	* @remarks Called with interrupts disabled.
856	*/
857	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
858	{
859	Assert(pVmcsInfoFrom);
860	Assert(pVmcsInfoTo);
861
862	/*
863	* Clear the VMCS we are switching out if it has not already been cleared.
864	* This will sync any CPU internal data back to the VMCS.
865	*/
866	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
867	{
868	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
869	if (RT_SUCCESS(rc))
870	{ /* likely */ }
871	else
872	return rc;
873	}
874
875	/*
876	* Clear the VMCS we are switching to if it has not already been cleared.
877	* This will initialize the VMCS launch state to "clear" required for loading it.
878	*
879	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
880	*/
881	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
882	{
883	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
884	if (RT_SUCCESS(rc))
885	{ /* likely */ }
886	else
887	return rc;
888	}
889
890	/*
891	* Finally, load the VMCS we are switching to.
892	*/
893	return hmR0VmxLoadVmcs(pVmcsInfoTo);
894	}
895	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
896
897
898	/**
899	* Updates the VM's last error record.
900	*
901	* If there was a VMX instruction error, reads the error data from the VMCS and
902	* updates VCPU's last error record as well.
903	*
904	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
905	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
906	* VERR_VMX_INVALID_VMCS_FIELD.
907	* @param rc The error code.
908	*/
909	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
910	{
911	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
912	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
913	{
914	AssertPtrReturnVoid(pVCpu);
915	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
916	}
917	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
918	}
919
920
921	#ifdef VBOX_STRICT
922	/**
923	* Reads the VM-entry interruption-information field from the VMCS into the VMX
924	* transient structure.
925	*
926	* @returns VBox status code.
927	* @param pVmxTransient The VMX-transient structure.
928	*
929	* @remarks No-long-jump zone!!!
930	*/
931	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
932	{
933	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
934	AssertRCReturn(rc, rc);
935	return VINF_SUCCESS;
936	}
937
938
939	/**
940	* Reads the VM-entry exception error code field from the VMCS into
941	* the VMX transient structure.
942	*
943	* @returns VBox status code.
944	* @param pVmxTransient The VMX-transient structure.
945	*
946	* @remarks No-long-jump zone!!!
947	*/
948	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
949	{
950	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
951	AssertRCReturn(rc, rc);
952	return VINF_SUCCESS;
953	}
954
955
956	/**
957	* Reads the VM-entry exception error code field from the VMCS into
958	* the VMX transient structure.
959	*
960	* @returns VBox status code.
961	* @param pVmxTransient The VMX-transient structure.
962	*
963	* @remarks No-long-jump zone!!!
964	*/
965	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
966	{
967	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
968	AssertRCReturn(rc, rc);
969	return VINF_SUCCESS;
970	}
971	#endif /* VBOX_STRICT */
972
973
974	/**
975	* Reads the VM-exit interruption-information field from the VMCS into the VMX
976	* transient structure.
977	*
978	* @returns VBox status code.
979	* @param pVmxTransient The VMX-transient structure.
980	*/
981	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
982	{
983	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
984	{
985	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
986	AssertRCReturn(rc,rc);
987	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
988	}
989	return VINF_SUCCESS;
990	}
991
992
993	/**
994	* Reads the VM-exit interruption error code from the VMCS into the VMX
995	* transient structure.
996	*
997	* @returns VBox status code.
998	* @param pVmxTransient The VMX-transient structure.
999	*/
1000	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1001	{
1002	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1003	{
1004	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1005	AssertRCReturn(rc, rc);
1006	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1007	}
1008	return VINF_SUCCESS;
1009	}
1010
1011
1012	/**
1013	* Reads the VM-exit instruction length field from the VMCS into the VMX
1014	* transient structure.
1015	*
1016	* @returns VBox status code.
1017	* @param pVmxTransient The VMX-transient structure.
1018	*/
1019	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1020	{
1021	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1022	{
1023	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1024	AssertRCReturn(rc, rc);
1025	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1026	}
1027	return VINF_SUCCESS;
1028	}
1029
1030
1031	/**
1032	* Reads the VM-exit instruction-information field from the VMCS into
1033	* the VMX transient structure.
1034	*
1035	* @returns VBox status code.
1036	* @param pVmxTransient The VMX-transient structure.
1037	*/
1038	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1039	{
1040	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1041	{
1042	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1043	AssertRCReturn(rc, rc);
1044	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1045	}
1046	return VINF_SUCCESS;
1047	}
1048
1049
1050	/**
1051	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1052	*
1053	* @returns VBox status code.
1054	* @param pVCpu The cross context virtual CPU structure of the
1055	* calling EMT. (Required for the VMCS cache case.)
1056	* @param pVmxTransient The VMX-transient structure.
1057	*/
1058	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1059	{
1060	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1061	{
1062	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1063	AssertRCReturn(rc, rc);
1064	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1065	}
1066	return VINF_SUCCESS;
1067	}
1068
1069
1070	/**
1071	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1072	*
1073	* @returns VBox status code.
1074	* @param pVCpu The cross context virtual CPU structure of the
1075	* calling EMT. (Required for the VMCS cache case.)
1076	* @param pVmxTransient The VMX-transient structure.
1077	*/
1078	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1079	{
1080	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1081	{
1082	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1083	AssertRCReturn(rc, rc);
1084	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1085	}
1086	return VINF_SUCCESS;
1087	}
1088
1089
1090	/**
1091	* Reads the IDT-vectoring information field from the VMCS into the VMX
1092	* transient structure.
1093	*
1094	* @returns VBox status code.
1095	* @param pVmxTransient The VMX-transient structure.
1096	*
1097	* @remarks No-long-jump zone!!!
1098	*/
1099	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1100	{
1101	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1102	{
1103	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1104	AssertRCReturn(rc, rc);
1105	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1106	}
1107	return VINF_SUCCESS;
1108	}
1109
1110
1111	/**
1112	* Reads the IDT-vectoring error code from the VMCS into the VMX
1113	* transient structure.
1114	*
1115	* @returns VBox status code.
1116	* @param pVmxTransient The VMX-transient structure.
1117	*/
1118	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1119	{
1120	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1121	{
1122	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1123	AssertRCReturn(rc, rc);
1124	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1125	}
1126	return VINF_SUCCESS;
1127	}
1128
1129
1130	/**
1131	* Enters VMX root mode operation on the current CPU.
1132	*
1133	* @returns VBox status code.
1134	* @param pVM The cross context VM structure. Can be
1135	* NULL, after a resume.
1136	* @param HCPhysCpuPage Physical address of the VMXON region.
1137	* @param pvCpuPage Pointer to the VMXON region.
1138	*/
1139	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1140	{
1141	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1142	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1143	Assert(pvCpuPage);
1144	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1145
1146	if (pVM)
1147	{
1148	/* Write the VMCS revision identifier to the VMXON region. */
1149	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1150	}
1151
1152	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1153	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1154
1155	/* Enable the VMX bit in CR4 if necessary. */
1156	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1157
1158	/* Enter VMX root mode. */
1159	int rc = VMXEnable(HCPhysCpuPage);
1160	if (RT_FAILURE(rc))
1161	{
1162	if (!(uOldCr4 & X86_CR4_VMXE))
1163	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1164
1165	if (pVM)
1166	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1167	}
1168
1169	/* Restore interrupts. */
1170	ASMSetFlags(fEFlags);
1171	return rc;
1172	}
1173
1174
1175	/**
1176	* Exits VMX root mode operation on the current CPU.
1177	*
1178	* @returns VBox status code.
1179	*/
1180	static int hmR0VmxLeaveRootMode(void)
1181	{
1182	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1183
1184	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1185	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1186
1187	/* If we're for some reason not in VMX root mode, then don't leave it. */
1188	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1189
1190	int rc;
1191	if (uHostCR4 & X86_CR4_VMXE)
1192	{
1193	/* Exit VMX root mode and clear the VMX bit in CR4. */
1194	VMXDisable();
1195	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1196	rc = VINF_SUCCESS;
1197	}
1198	else
1199	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1200
1201	/* Restore interrupts. */
1202	ASMSetFlags(fEFlags);
1203	return rc;
1204	}
1205
1206
1207	/**
1208	* Allocates and maps a physically contiguous page. The allocated page is
1209	* zero'd out (used by various VT-x structures).
1210	*
1211	* @returns IPRT status code.
1212	* @param pMemObj Pointer to the ring-0 memory object.
1213	* @param ppVirt Where to store the virtual address of the
1214	* allocation.
1215	* @param pHCPhys Where to store the physical address of the
1216	* allocation.
1217	*/
1218	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1219	{
1220	AssertPtr(pMemObj);
1221	AssertPtr(ppVirt);
1222	AssertPtr(pHCPhys);
1223	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1224	if (RT_FAILURE(rc))
1225	return rc;
1226	ppVirt = RTR0MemObjAddress(pMemObj);
1227	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1228	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Frees and unmaps an allocated, physical page.
1235	*
1236	* @param pMemObj Pointer to the ring-0 memory object.
1237	* @param ppVirt Where to re-initialize the virtual address of
1238	* allocation as 0.
1239	* @param pHCPhys Where to re-initialize the physical address of the
1240	* allocation as 0.
1241	*/
1242	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1243	{
1244	AssertPtr(pMemObj);
1245	AssertPtr(ppVirt);
1246	AssertPtr(pHCPhys);
1247	/* NULL is valid, accepted and ignored by the free function below. */
1248	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1249	*pMemObj = NIL_RTR0MEMOBJ;
1250	*ppVirt = NULL;
1251	*pHCPhys = NIL_RTHCPHYS;
1252	}
1253
1254
1255	/**
1256	* Initializes a VMCS info. object.
1257	*
1258	* @param pVmcsInfo The VMCS info. object.
1259	*/
1260	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1261	{
1262	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1263
1264	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1265	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1266	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1267	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1268	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1269	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1272	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1273	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1274	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1275	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1276	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1277	}
1278
1279
1280	/**
1281	* Frees the VT-x structures for a VMCS info. object.
1282	*
1283	* @param pVM The cross context VM structure.
1284	* @param pVmcsInfo The VMCS info. object.
1285	*/
1286	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1287	{
1288	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1289
1290	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1292
1293	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1294	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1295	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1296
1297	hmR0VmxInitVmcsInfo(pVmcsInfo);
1298	}
1299
1300
1301	/**
1302	* Allocates the VT-x structures for a VMCS info. object.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu The cross context virtual CPU structure.
1306	* @param pVmcsInfo The VMCS info. object.
1307	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1308	*/
1309	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1310	{
1311	PVM pVM = pVCpu->CTX_SUFF(pVM);
1312
1313	/* Allocate the guest VM control structure (VMCS). */
1314	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1315	if (RT_SUCCESS(rc))
1316	{
1317	if (!fIsNstGstVmcs)
1318	{
1319	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1320	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1321	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1322	{
1323	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1324	NULL /* pR3Ptr /, NULL / pRCPtr */);
1325	}
1326	}
1327	else
1328	{
1329	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1330	Assert(!pVmcsInfo->pbVirtApic);
1331	}
1332
1333	if (RT_SUCCESS(rc))
1334	{
1335	/*
1336	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1337	* transparent accesses of specific MSRs.
1338	*
1339	* If the condition for enabling MSR bitmaps changes here, don't forget to
1340	* update HMIsMsrBitmapActive().
1341	*
1342	* We don't share MSR bitmaps between the guest and nested-guest as we then
1343	* don't need to care about carefully restoring the guest MSR bitmap.
1344	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1345	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1346	*/
1347	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1348	{
1349	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1350	if (RT_SUCCESS(rc))
1351	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1352	}
1353
1354	if (RT_SUCCESS(rc))
1355	{
1356	/*
1357	* Allocate the VM-entry MSR-load area for the guest MSRs.
1358	*
1359	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1360	* the guest and nested-guest.
1361	*/
1362	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1363	&pVmcsInfo->HCPhysGuestMsrLoad);
1364	if (RT_SUCCESS(rc))
1365	{
1366	/*
1367	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1368	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1369	*/
1370	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1371	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1372	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1373
1374	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1375	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1376	&pVmcsInfo->HCPhysHostMsrLoad);
1377	}
1378	}
1379	}
1380	}
1381
1382	return rc;
1383	}
1384
1385
1386	/**
1387	* Free all VT-x structures for the VM.
1388	*
1389	* @returns IPRT status code.
1390	* @param pVM The cross context VM structure.
1391	*/
1392	static void hmR0VmxStructsFree(PVM pVM)
1393	{
1394	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1395	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1396	#endif
1397	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1398
1399	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1400	{
1401	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1402	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1403	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1404	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1405	if (pVM->cpum.ro.GuestFeatures.fVmx)
1406	{
1407	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1408	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1409	}
1410	#endif
1411	}
1412	}
1413
1414
1415	/**
1416	* Allocate all VT-x structures for the VM.
1417	*
1418	* @returns IPRT status code.
1419	* @param pVM The cross context VM structure.
1420	*/
1421	static int hmR0VmxStructsAlloc(PVM pVM)
1422	{
1423	/*
1424	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1425	* The VMCS size cannot be more than 4096 bytes.
1426	*
1427	* See Intel spec. Appendix A.1 "Basic VMX Information".
1428	*/
1429	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1430	if (cbVmcs <= X86_PAGE_4K_SIZE)
1431	{ /* likely */ }
1432	else
1433	{
1434	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1435	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1436	}
1437
1438	/*
1439	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1440	*/
1441	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1442	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1443	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1444	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1445	#endif
1446
1447	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1448	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1449	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1450
1451	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1452	{
1453	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1454	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1455	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1456	}
1457
1458	/*
1459	* Allocate per-VM VT-x structures.
1460	*/
1461	int rc = VINF_SUCCESS;
1462	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1463	/* Allocate crash-dump magic scratch page. */
1464	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1465	if (RT_FAILURE(rc))
1466	{
1467	hmR0VmxStructsFree(pVM);
1468	return rc;
1469	}
1470	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1471	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1472	#endif
1473
1474	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1475	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1476	{
1477	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1478	&pVM->hm.s.vmx.HCPhysApicAccess);
1479	if (RT_FAILURE(rc))
1480	{
1481	hmR0VmxStructsFree(pVM);
1482	return rc;
1483	}
1484	}
1485
1486	/*
1487	* Initialize per-VCPU VT-x structures.
1488	*/
1489	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1490	{
1491	/* Allocate the guest VMCS structures. */
1492	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1493	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1494	if (RT_SUCCESS(rc))
1495	{
1496	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1497	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1498	if (pVM->cpum.ro.GuestFeatures.fVmx)
1499	{
1500	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1501	if (RT_SUCCESS(rc))
1502	{ /* likely */ }
1503	else
1504	break;
1505	}
1506	#endif
1507	}
1508	else
1509	break;
1510	}
1511
1512	if (RT_FAILURE(rc))
1513	{
1514	hmR0VmxStructsFree(pVM);
1515	return rc;
1516	}
1517
1518	return VINF_SUCCESS;
1519	}
1520
1521
1522	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1523	/**
1524	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1525	*
1526	* @returns @c true if the MSR is intercepted, @c false otherwise.
1527	* @param pvMsrBitmap The MSR bitmap.
1528	* @param offMsr The MSR byte offset.
1529	* @param iBit The bit offset from the byte offset.
1530	*/
1531	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1532	{
1533	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1534	Assert(pbMsrBitmap);
1535	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1536	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1537	}
1538	#endif
1539
1540
1541	/**
1542	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1543	*
1544	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1545	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1546	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1547	* the read/write access of this MSR.
1548	*
1549	* @param pVCpu The cross context virtual CPU structure.
1550	* @param pVmcsInfo The VMCS info. object.
1551	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1552	* @param idMsr The MSR value.
1553	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1554	* include both a read -and- a write permission!
1555	*
1556	* @sa CPUMGetVmxMsrPermission.
1557	* @remarks Can be called with interrupts disabled.
1558	*/
1559	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1560	{
1561	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1562	Assert(pbMsrBitmap);
1563	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1564
1565	/*
1566	* MSR-bitmap Layout:
1567	* Byte index MSR range Interpreted as
1568	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1569	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1570	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1571	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1572	*
1573	* A bit corresponding to an MSR within the above range causes a VM-exit
1574	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1575	* the MSR range, it always cause a VM-exit.
1576	*
1577	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1578	*/
1579	uint16_t const offBitmapRead = 0;
1580	uint16_t const offBitmapWrite = 0x800;
1581	uint16_t offMsr;
1582	int32_t iBit;
1583	if (idMsr <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0;
1586	iBit = idMsr;
1587	}
1588	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1589	{
1590	offMsr = 0x400;
1591	iBit = idMsr - UINT32_C(0xc0000000);
1592	}
1593	else
1594	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1595
1596	/*
1597	* Set the MSR read permission.
1598	*/
1599	uint16_t const offMsrRead = offBitmapRead + offMsr;
1600	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1601	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1602	{
1603	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1604	bool const fClear = !fIsNstGstVmcs ? true
1605	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1606	#else
1607	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1608	bool const fClear = true;
1609	#endif
1610	if (fClear)
1611	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1612	}
1613	else
1614	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1615
1616	/*
1617	* Set the MSR write permission.
1618	*/
1619	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1620	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1621	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1622	{
1623	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1624	bool const fClear = !fIsNstGstVmcs ? true
1625	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1626	#else
1627	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1628	bool const fClear = true;
1629	#endif
1630	if (fClear)
1631	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1632	}
1633	else
1634	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1635	}
1636
1637
1638	/**
1639	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1640	* area.
1641	*
1642	* @returns VBox status code.
1643	* @param pVCpu The cross context virtual CPU structure.
1644	* @param pVmcsInfo The VMCS info. object.
1645	* @param cMsrs The number of MSRs.
1646	*/
1647	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1648	{
1649	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1650	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1651	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1652	{
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1655	{
1656	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1659	AssertRCReturn(rc, rc);
1660
1661	pVmcsInfo->cEntryMsrLoad = cMsrs;
1662	pVmcsInfo->cExitMsrStore = cMsrs;
1663	pVmcsInfo->cExitMsrLoad = cMsrs;
1664	}
1665	return VINF_SUCCESS;
1666	}
1667
1668	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1669	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1670	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1671	}
1672
1673
1674	/**
1675	* Adds a new (or updates the value of an existing) guest/host MSR
1676	* pair to be swapped during the world-switch as part of the
1677	* auto-load/store MSR area in the VMCS.
1678	*
1679	* @returns VBox status code.
1680	* @param pVCpu The cross context virtual CPU structure.
1681	* @param pVmxTransient The VMX-transient structure.
1682	* @param idMsr The MSR.
1683	* @param uGuestMsrValue Value of the guest MSR.
1684	* @param fSetReadWrite Whether to set the guest read/write access of this
1685	* MSR (thus not causing a VM-exit).
1686	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1687	* necessary.
1688	*/
1689	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1690	bool fSetReadWrite, bool fUpdateHostMsr)
1691	{
1692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1693	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1694	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1695	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1696	uint32_t i;
1697
1698	/* Paranoia. */
1699	Assert(pGuestMsrLoad);
1700
1701	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1702
1703	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1704	for (i = 0; i < cMsrs; i++)
1705	{
1706	if (pGuestMsrLoad[i].u32Msr == idMsr)
1707	break;
1708	}
1709
1710	bool fAdded = false;
1711	if (i == cMsrs)
1712	{
1713	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1714	++cMsrs;
1715	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1716	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1717
1718	/* Set the guest to read/write this MSR without causing VM-exits. */
1719	if ( fSetReadWrite
1720	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1721	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1722
1723	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1724	fAdded = true;
1725	}
1726
1727	/* Update the MSR value for the newly added or already existing MSR. */
1728	pGuestMsrLoad[i].u32Msr = idMsr;
1729	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1730
1731	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1732	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1733	{
1734	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1735	pGuestMsrStore[i].u32Msr = idMsr;
1736	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1737	}
1738
1739	/* Update the corresponding slot in the host MSR area. */
1740	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1741	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1742	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1743	pHostMsr[i].u32Msr = idMsr;
1744
1745	/*
1746	* Only if the caller requests to update the host MSR value AND we've newly added the
1747	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1748	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1749	*
1750	* We do this for performance reasons since reading MSRs may be quite expensive.
1751	*/
1752	if (fAdded)
1753	{
1754	if (fUpdateHostMsr)
1755	{
1756	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1757	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1758	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1759	}
1760	else
1761	{
1762	/* Someone else can do the work. */
1763	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1764	}
1765	}
1766	return VINF_SUCCESS;
1767	}
1768
1769
1770	/**
1771	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1772	* auto-load/store MSR area in the VMCS.
1773	*
1774	* @returns VBox status code.
1775	* @param pVCpu The cross context virtual CPU structure.
1776	* @param pVmxTransient The VMX-transient structure.
1777	* @param idMsr The MSR.
1778	*/
1779	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1780	{
1781	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1782	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1783	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1784	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1785
1786	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1787
1788	for (uint32_t i = 0; i < cMsrs; i++)
1789	{
1790	/* Find the MSR. */
1791	if (pGuestMsrLoad[i].u32Msr == idMsr)
1792	{
1793	/*
1794	* If it's the last MSR, we only need to reduce the MSR count.
1795	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1796	*/
1797	if (i < cMsrs - 1)
1798	{
1799	/* Remove it from the VM-entry MSR-load area. */
1800	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1801	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1802
1803	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1804	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1805	{
1806	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1807	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1808	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1809	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1810	}
1811
1812	/* Remove it from the VM-exit MSR-load area. */
1813	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1814	Assert(pHostMsr[i].u32Msr == idMsr);
1815	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1816	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1817	}
1818
1819	/* Reduce the count to reflect the removed MSR and bail. */
1820	--cMsrs;
1821	break;
1822	}
1823	}
1824
1825	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1826	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1827	{
1828	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1829	AssertRCReturn(rc, rc);
1830
1831	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1832	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1833	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1834
1835	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1836	return VINF_SUCCESS;
1837	}
1838
1839	return VERR_NOT_FOUND;
1840	}
1841
1842
1843	/**
1844	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1845	*
1846	* @returns @c true if found, @c false otherwise.
1847	* @param pVmcsInfo The VMCS info. object.
1848	* @param idMsr The MSR to find.
1849	*/
1850	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1851	{
1852	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1853	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1854	Assert(pMsrs);
1855	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1856	for (uint32_t i = 0; i < cMsrs; i++)
1857	{
1858	if (pMsrs[i].u32Msr == idMsr)
1859	return true;
1860	}
1861	return false;
1862	}
1863
1864
1865	/**
1866	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1867	*
1868	* @param pVCpu The cross context virtual CPU structure.
1869	* @param pVmcsInfo The VMCS info. object.
1870	*
1871	* @remarks No-long-jump zone!!!
1872	*/
1873	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1874	{
1875	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1876
1877	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1878	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1879	Assert(pHostMsrLoad);
1880	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1881	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1882	for (uint32_t i = 0; i < cMsrs; i++)
1883	{
1884	/*
1885	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1886	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1887	*/
1888	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1889	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1890	else
1891	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1892	}
1893	}
1894
1895
1896	/**
1897	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1898	* perform lazy restoration of the host MSRs while leaving VT-x.
1899	*
1900	* @param pVCpu The cross context virtual CPU structure.
1901	*
1902	* @remarks No-long-jump zone!!!
1903	*/
1904	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1905	{
1906	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1907
1908	/*
1909	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1910	*/
1911	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1912	{
1913	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1914	#if HC_ARCH_BITS == 64
1915	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1916	{
1917	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1918	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1919	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1920	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1921	}
1922	#endif
1923	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1924	}
1925	}
1926
1927
1928	/**
1929	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1930	* lazily while leaving VT-x.
1931	*
1932	* @returns true if it does, false otherwise.
1933	* @param pVCpu The cross context virtual CPU structure.
1934	* @param idMsr The MSR to check.
1935	*/
1936	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1937	{
1938	NOREF(pVCpu);
1939	#if HC_ARCH_BITS == 64
1940	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1941	{
1942	switch (idMsr)
1943	{
1944	case MSR_K8_LSTAR:
1945	case MSR_K6_STAR:
1946	case MSR_K8_SF_MASK:
1947	case MSR_K8_KERNEL_GS_BASE:
1948	return true;
1949	}
1950	}
1951	#else
1952	RT_NOREF(pVCpu, idMsr);
1953	#endif
1954	return false;
1955	}
1956
1957
1958	/**
1959	* Loads a set of guests MSRs to allow read/passthru to the guest.
1960	*
1961	* The name of this function is slightly confusing. This function does NOT
1962	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1963	* common prefix for functions dealing with "lazy restoration" of the shared
1964	* MSRs.
1965	*
1966	* @param pVCpu The cross context virtual CPU structure.
1967	*
1968	* @remarks No-long-jump zone!!!
1969	*/
1970	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1971	{
1972	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1973	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1974
1975	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1976	#if HC_ARCH_BITS == 64
1977	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1978	{
1979	/*
1980	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1981	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1982	* we can skip a few MSR writes.
1983	*
1984	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1985	* guest MSR values in the guest-CPU context might be different to what's currently
1986	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1987	* CPU, see @bugref{8728}.
1988	*/
1989	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1990	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1991	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1992	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1993	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1994	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1995	{
1996	#ifdef VBOX_STRICT
1997	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1998	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1999	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2000	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2001	#endif
2002	}
2003	else
2004	{
2005	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2006	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2007	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2008	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2009	}
2010	}
2011	#endif
2012	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2013	}
2014
2015
2016	/**
2017	* Performs lazy restoration of the set of host MSRs if they were previously
2018	* loaded with guest MSR values.
2019	*
2020	* @param pVCpu The cross context virtual CPU structure.
2021	*
2022	* @remarks No-long-jump zone!!!
2023	* @remarks The guest MSRs should have been saved back into the guest-CPU
2024	* context by hmR0VmxImportGuestState()!!!
2025	*/
2026	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2027	{
2028	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2029	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2030
2031	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2032	{
2033	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2034	#if HC_ARCH_BITS == 64
2035	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2036	{
2037	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2038	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2039	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2040	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2041	}
2042	#endif
2043	}
2044	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2045	}
2046
2047
2048	/**
2049	* Verifies that our cached values of the VMCS fields are all consistent with
2050	* what's actually present in the VMCS.
2051	*
2052	* @returns VBox status code.
2053	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2054	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2055	* VMCS content. HMCPU error-field is
2056	* updated, see VMX_VCI_XXX.
2057	* @param pVCpu The cross context virtual CPU structure.
2058	* @param pVmcsInfo The VMCS info. object.
2059	*/
2060	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2061	{
2062	uint32_t u32Val;
2063	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2085	AssertRCReturn(rc, rc);
2086	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2087	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2088	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2089	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2090
2091	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2092	{
2093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2094	AssertRCReturn(rc, rc);
2095	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2096	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2097	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2098	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2099	}
2100
2101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2102	AssertRCReturn(rc, rc);
2103	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2104	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2105	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2106	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2107
2108	uint64_t u64Val;
2109	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2110	AssertRCReturn(rc, rc);
2111	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2112	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2113	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2114	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2115
2116	return VINF_SUCCESS;
2117	}
2118
2119
2120	#ifdef VBOX_STRICT
2121	/**
2122	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2123	*
2124	* @param pVCpu The cross context virtual CPU structure.
2125	* @param pVmcsInfo The VMCS info. object.
2126	*/
2127	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2128	{
2129	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2130
2131	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2132	{
2133	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2134	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2135	uint64_t uVmcsEferMsrVmcs;
2136	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2137	AssertRC(rc);
2138
2139	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2140	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2141	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2142	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2143	}
2144	}
2145
2146
2147	/**
2148	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2149	* VMCS are correct.
2150	*
2151	* @param pVCpu The cross context virtual CPU structure.
2152	* @param pVmcsInfo The VMCS info. object.
2153	*/
2154	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2155	{
2156	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2157
2158	/* Read the various MSR-area counts from the VMCS. */
2159	uint32_t cEntryLoadMsrs;
2160	uint32_t cExitStoreMsrs;
2161	uint32_t cExitLoadMsrs;
2162	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2163	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2164	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2165
2166	/* Verify all the MSR counts are the same. */
2167	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2168	Assert(cExitStoreMsrs == cExitLoadMsrs);
2169	uint32_t const cMsrs = cExitLoadMsrs;
2170
2171	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2172	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2173
2174	/* Verify the MSR counts are within the allocated page size. */
2175	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2176
2177	/* Verify the relevant contents of the MSR areas match. */
2178	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2179	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2180	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2181	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2182	for (uint32_t i = 0; i < cMsrs; i++)
2183	{
2184	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2185	if (fSeparateExitMsrStorePage)
2186	{
2187	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2188	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2189	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2190	}
2191
2192	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2193	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2194	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2195
2196	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2197	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2198	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2199	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2200
2201	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2202	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2203	if (fIsEferMsr)
2204	{
2205	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2206	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2207	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2208	}
2209
2210	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2211	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2212	{
2213	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2214	if (fIsEferMsr)
2215	{
2216	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2217	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2218	}
2219	else
2220	{
2221	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2222	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2223	}
2224	}
2225
2226	/* Move to the next MSR. */
2227	pHostMsrLoad++;
2228	pGuestMsrLoad++;
2229	pGuestMsrStore++;
2230	}
2231	}
2232	#endif /* VBOX_STRICT */
2233
2234
2235	/**
2236	* Flushes the TLB using EPT.
2237	*
2238	* @returns VBox status code.
2239	* @param pVCpu The cross context virtual CPU structure of the calling
2240	* EMT. Can be NULL depending on @a enmTlbFlush.
2241	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2242	* enmTlbFlush.
2243	* @param enmTlbFlush Type of flush.
2244	*
2245	* @remarks Caller is responsible for making sure this function is called only
2246	* when NestedPaging is supported and providing @a enmTlbFlush that is
2247	* supported by the CPU.
2248	* @remarks Can be called with interrupts disabled.
2249	*/
2250	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2251	{
2252	uint64_t au64Descriptor[2];
2253	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2254	au64Descriptor[0] = 0;
2255	else
2256	{
2257	Assert(pVCpu);
2258	Assert(pVmcsInfo);
2259	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2260	}
2261	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2262
2263	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2264	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2265
2266	if ( RT_SUCCESS(rc)
2267	&& pVCpu)
2268	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2269	}
2270
2271
2272	/**
2273	* Flushes the TLB using VPID.
2274	*
2275	* @returns VBox status code.
2276	* @param pVCpu The cross context virtual CPU structure of the calling
2277	* EMT. Can be NULL depending on @a enmTlbFlush.
2278	* @param enmTlbFlush Type of flush.
2279	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2280	* on @a enmTlbFlush).
2281	*
2282	* @remarks Can be called with interrupts disabled.
2283	*/
2284	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2285	{
2286	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2287
2288	uint64_t au64Descriptor[2];
2289	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2290	{
2291	au64Descriptor[0] = 0;
2292	au64Descriptor[1] = 0;
2293	}
2294	else
2295	{
2296	AssertPtr(pVCpu);
2297	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2298	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2299	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2300	au64Descriptor[1] = GCPtr;
2301	}
2302
2303	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2304	AssertMsg(rc == VINF_SUCCESS,
2305	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2306
2307	if ( RT_SUCCESS(rc)
2308	&& pVCpu)
2309	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2310	NOREF(rc);
2311	}
2312
2313
2314	/**
2315	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2316	* otherwise there is nothing really to invalidate.
2317	*
2318	* @returns VBox status code.
2319	* @param pVCpu The cross context virtual CPU structure.
2320	* @param GCVirt Guest virtual address of the page to invalidate.
2321	*/
2322	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2323	{
2324	AssertPtr(pVCpu);
2325	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2326
2327	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2328	{
2329	/*
2330	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2331	* the EPT case. See @bugref{6043} and @bugref{6177}.
2332	*
2333	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2334	* as this function maybe called in a loop with individual addresses.
2335	*/
2336	PVM pVM = pVCpu->CTX_SUFF(pVM);
2337	if (pVM->hm.s.vmx.fVpid)
2338	{
2339	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2340
2341	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2342	/*
2343	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2344	* where executing INVVPID outside 64-bit mode does not flush translations of
2345	* 64-bit linear addresses, see @bugref{6208#c72}.
2346	*/
2347	if (RT_HI_U32(GCVirt))
2348	fVpidFlush = false;
2349	#endif
2350
2351	if (fVpidFlush)
2352	{
2353	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2354	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2355	}
2356	else
2357	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2358	}
2359	else if (pVM->hm.s.fNestedPaging)
2360	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2361	}
2362
2363	return VINF_SUCCESS;
2364	}
2365
2366
2367	/**
2368	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2369	* case where neither EPT nor VPID is supported by the CPU.
2370	*
2371	* @param pHostCpu The HM physical-CPU structure.
2372	* @param pVCpu The cross context virtual CPU structure.
2373	*
2374	* @remarks Called with interrupts disabled.
2375	*/
2376	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2377	{
2378	AssertPtr(pVCpu);
2379	AssertPtr(pHostCpu);
2380
2381	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2382
2383	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2384	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2385	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2386	pVCpu->hm.s.fForceTLBFlush = false;
2387	return;
2388	}
2389
2390
2391	/**
2392	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2393	*
2394	* @param pHostCpu The HM physical-CPU structure.
2395	* @param pVCpu The cross context virtual CPU structure.
2396	* @param pVmcsInfo The VMCS info. object.
2397	*
2398	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2399	* nomenclature. The reason is, to avoid confusion in compare statements
2400	* since the host-CPU copies are named "ASID".
2401	*
2402	* @remarks Called with interrupts disabled.
2403	*/
2404	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2405	{
2406	#ifdef VBOX_WITH_STATISTICS
2407	bool fTlbFlushed = false;
2408	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2409	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2410	if (!fTlbFlushed) \
2411	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2412	} while (0)
2413	#else
2414	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2415	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2416	#endif
2417
2418	AssertPtr(pVCpu);
2419	AssertPtr(pHostCpu);
2420	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2421
2422	PVM pVM = pVCpu->CTX_SUFF(pVM);
2423	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2424	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2425	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2426
2427	/*
2428	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2429	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2430	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2431	* cannot reuse the current ASID anymore.
2432	*/
2433	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2434	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2435	{
2436	++pHostCpu->uCurrentAsid;
2437	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2438	{
2439	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2440	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2441	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2442	}
2443
2444	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2445	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2446	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2447
2448	/*
2449	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2450	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2451	*/
2452	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2453	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2454	HMVMX_SET_TAGGED_TLB_FLUSHED();
2455	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2456	}
2457	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2458	{
2459	/*
2460	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2461	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2462	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2463	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2464	* mappings, see @bugref{6568}.
2465	*
2466	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2467	*/
2468	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2469	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2470	HMVMX_SET_TAGGED_TLB_FLUSHED();
2471	}
2472
2473	pVCpu->hm.s.fForceTLBFlush = false;
2474	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2475
2476	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2477	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2478	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2479	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2480	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2481	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2482	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2483	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2484	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2485
2486	/* Update VMCS with the VPID. */
2487	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2488	AssertRC(rc);
2489
2490	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2491	}
2492
2493
2494	/**
2495	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2496	*
2497	* @param pHostCpu The HM physical-CPU structure.
2498	* @param pVCpu The cross context virtual CPU structure.
2499	* @param pVmcsInfo The VMCS info. object.
2500	*
2501	* @remarks Called with interrupts disabled.
2502	*/
2503	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2504	{
2505	AssertPtr(pVCpu);
2506	AssertPtr(pHostCpu);
2507	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2508	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2509	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2510
2511	/*
2512	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2513	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2514	*/
2515	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2516	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2517	{
2518	pVCpu->hm.s.fForceTLBFlush = true;
2519	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2520	}
2521
2522	/* Check for explicit TLB flushes. */
2523	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2524	{
2525	pVCpu->hm.s.fForceTLBFlush = true;
2526	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2527	}
2528
2529	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2530	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2531
2532	if (pVCpu->hm.s.fForceTLBFlush)
2533	{
2534	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2535	pVCpu->hm.s.fForceTLBFlush = false;
2536	}
2537	}
2538
2539
2540	/**
2541	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2542	*
2543	* @param pHostCpu The HM physical-CPU structure.
2544	* @param pVCpu The cross context virtual CPU structure.
2545	*
2546	* @remarks Called with interrupts disabled.
2547	*/
2548	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2549	{
2550	AssertPtr(pVCpu);
2551	AssertPtr(pHostCpu);
2552	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2553	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2554	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2555
2556	/*
2557	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2558	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2559	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2560	* cannot reuse the current ASID anymore.
2561	*/
2562	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2563	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2564	{
2565	pVCpu->hm.s.fForceTLBFlush = true;
2566	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2567	}
2568
2569	/* Check for explicit TLB flushes. */
2570	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2571	{
2572	/*
2573	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2574	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2575	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2576	* include fExplicitFlush's too) - an obscure corner case.
2577	*/
2578	pVCpu->hm.s.fForceTLBFlush = true;
2579	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2580	}
2581
2582	PVM pVM = pVCpu->CTX_SUFF(pVM);
2583	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2584	if (pVCpu->hm.s.fForceTLBFlush)
2585	{
2586	++pHostCpu->uCurrentAsid;
2587	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2588	{
2589	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2590	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2591	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2592	}
2593
2594	pVCpu->hm.s.fForceTLBFlush = false;
2595	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2596	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2597	if (pHostCpu->fFlushAsidBeforeUse)
2598	{
2599	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2600	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2601	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2602	{
2603	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2604	pHostCpu->fFlushAsidBeforeUse = false;
2605	}
2606	else
2607	{
2608	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2609	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2610	}
2611	}
2612	}
2613
2614	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2615	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2616	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2617	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2618	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2619	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2620	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2621
2622	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2623	AssertRC(rc);
2624	}
2625
2626
2627	/**
2628	* Flushes the guest TLB entry based on CPU capabilities.
2629	*
2630	* @param pHostCpu The HM physical-CPU structure.
2631	* @param pVCpu The cross context virtual CPU structure.
2632	* @param pVmcsInfo The VMCS info. object.
2633	*
2634	* @remarks Called with interrupts disabled.
2635	*/
2636	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2637	{
2638	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2639	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2640	#endif
2641	PVM pVM = pVCpu->CTX_SUFF(pVM);
2642	switch (pVM->hm.s.vmx.enmTlbFlushType)
2643	{
2644	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2645	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2646	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2647	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2648	default:
2649	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2650	break;
2651	}
2652	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2653	}
2654
2655
2656	/**
2657	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2658	* TLB entries from the host TLB before VM-entry.
2659	*
2660	* @returns VBox status code.
2661	* @param pVM The cross context VM structure.
2662	*/
2663	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2664	{
2665	/*
2666	* Determine optimal flush type for nested paging.
2667	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2668	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2669	*/
2670	if (pVM->hm.s.fNestedPaging)
2671	{
2672	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2673	{
2674	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2675	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2676	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2677	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2678	else
2679	{
2680	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2681	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2682	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2683	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2684	}
2685
2686	/* Make sure the write-back cacheable memory type for EPT is supported. */
2687	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2688	{
2689	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2690	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2691	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2692	}
2693
2694	/* EPT requires a page-walk length of 4. */
2695	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2696	{
2697	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2698	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2699	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2700	}
2701	}
2702	else
2703	{
2704	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2705	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2706	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2707	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2708	}
2709	}
2710
2711	/*
2712	* Determine optimal flush type for VPID.
2713	*/
2714	if (pVM->hm.s.vmx.fVpid)
2715	{
2716	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2717	{
2718	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2720	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2721	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2722	else
2723	{
2724	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2725	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2726	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2727	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2728	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2729	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2730	pVM->hm.s.vmx.fVpid = false;
2731	}
2732	}
2733	else
2734	{
2735	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2736	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2737	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2738	pVM->hm.s.vmx.fVpid = false;
2739	}
2740	}
2741
2742	/*
2743	* Setup the handler for flushing tagged-TLBs.
2744	*/
2745	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2746	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2747	else if (pVM->hm.s.fNestedPaging)
2748	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2749	else if (pVM->hm.s.vmx.fVpid)
2750	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2751	else
2752	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2753	return VINF_SUCCESS;
2754	}
2755
2756
2757	/**
2758	* Sets up the virtual-APIC page address for the VMCS.
2759	*
2760	* @returns VBox status code.
2761	* @param pVCpu The cross context virtual CPU structure.
2762	* @param pVmcsInfo The VMCS info. object.
2763	*/
2764	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2765	{
2766	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2767	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2768	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2769	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2770	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2771	}
2772
2773
2774	/**
2775	* Sets up the MSR-bitmap address for the VMCS.
2776	*
2777	* @returns VBox status code.
2778	* @param pVCpu The cross context virtual CPU structure.
2779	* @param pVmcsInfo The VMCS info. object.
2780	*/
2781	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2782	{
2783	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2784	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2785	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2786	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2787	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2788	}
2789
2790
2791	/**
2792	* Sets up the APIC-access page address for the VMCS.
2793	*
2794	* @returns VBox status code.
2795	* @param pVCpu The cross context virtual CPU structure.
2796	*/
2797	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2798	{
2799	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2800	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2801	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2802	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2803	}
2804
2805
2806	/**
2807	* Sets up the VMCS link pointer for the VMCS.
2808	*
2809	* @returns VBox status code.
2810	* @param pVCpu The cross context virtual CPU structure.
2811	* @param pVmcsInfo The VMCS info. object.
2812	*/
2813	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2814	{
2815	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2816	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2817	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2818	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2819	}
2820
2821
2822	/**
2823	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2824	* in the VMCS.
2825	*
2826	* @returns VBox status code.
2827	* @param pVCpu The cross context virtual CPU structure.
2828	* @param pVmcsInfo The VMCS info. object.
2829	*/
2830	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2831	{
2832	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2833
2834	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2835	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2836	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2837
2838	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2839	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2840	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2841
2842	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2843	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2844	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2845
2846	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2847	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2848	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2849	AssertRCReturn(rc, rc);
2850	return VINF_SUCCESS;
2851	}
2852
2853
2854	/**
2855	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2856	*
2857	* @param pVCpu The cross context virtual CPU structure.
2858	* @param pVmcsInfo The VMCS info. object.
2859	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2860	*/
2861	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2862	{
2863	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2864
2865	/*
2866	* The guest can access the following MSRs (read, write) without causing
2867	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2868	*/
2869	PVM pVM = pVCpu->CTX_SUFF(pVM);
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2871	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2872	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2873	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2874	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2875
2876	/*
2877	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2878	* associated with then. We never need to intercept access (writes need to be
2879	* executed without causing a VM-exit, reads will #GP fault anyway).
2880	*
2881	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2882	* read/write them. We swap the the guest/host MSR value using the
2883	* auto-load/store MSR area.
2884	*/
2885	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2886	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2887	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2888	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2889	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2890	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2891
2892	#if HC_ARCH_BITS == 64
2893	/*
2894	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2895	* required for 64-bit guests.
2896	*/
2897	if (pVM->hm.s.fAllow64BitGuests)
2898	{
2899	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2900	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2901	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2902	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2903	}
2904	#endif
2905
2906	/*
2907	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2908	*/
2909	#ifdef VBOX_STRICT
2910	Assert(pVmcsInfo->pvMsrBitmap);
2911	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2912	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2913	#endif
2914	}
2915
2916
2917	/**
2918	* Sets up pin-based VM-execution controls in the VMCS.
2919	*
2920	* @returns VBox status code.
2921	* @param pVCpu The cross context virtual CPU structure.
2922	* @param pVmcsInfo The VMCS info. object.
2923	*/
2924	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2925	{
2926	PVM pVM = pVCpu->CTX_SUFF(pVM);
2927	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2928	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2929
2930	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2931	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2932
2933	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2934	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2935
2936	/* Enable the VMX-preemption timer. */
2937	if (pVM->hm.s.vmx.fUsePreemptTimer)
2938	{
2939	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2940	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2941	}
2942
2943	#if 0
2944	/* Enable posted-interrupt processing. */
2945	if (pVM->hm.s.fPostedIntrs)
2946	{
2947	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2948	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2949	fVal \|= VMX_PIN_CTL_POSTED_INT;
2950	}
2951	#endif
2952
2953	if ((fVal & fZap) != fVal)
2954	{
2955	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2956	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2957	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2958	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2959	}
2960
2961	/* Commit it to the VMCS and update our cache. */
2962	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2963	AssertRCReturn(rc, rc);
2964	pVmcsInfo->u32PinCtls = fVal;
2965
2966	return VINF_SUCCESS;
2967	}
2968
2969
2970	/**
2971	* Sets up secondary processor-based VM-execution controls in the VMCS.
2972	*
2973	* @returns VBox status code.
2974	* @param pVCpu The cross context virtual CPU structure.
2975	* @param pVmcsInfo The VMCS info. object.
2976	*/
2977	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2978	{
2979	PVM pVM = pVCpu->CTX_SUFF(pVM);
2980	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2981	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2982
2983	/* WBINVD causes a VM-exit. */
2984	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2985	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2986
2987	/* Enable EPT (aka nested-paging). */
2988	if (pVM->hm.s.fNestedPaging)
2989	fVal \|= VMX_PROC_CTLS2_EPT;
2990
2991	/* Enable the INVPCID instruction if supported by the hardware and we expose
2992	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2993	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2994	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2995	fVal \|= VMX_PROC_CTLS2_INVPCID;
2996
2997	/* Enable VPID. */
2998	if (pVM->hm.s.vmx.fVpid)
2999	fVal \|= VMX_PROC_CTLS2_VPID;
3000
3001	/* Enable unrestricted guest execution. */
3002	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3003	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3004
3005	#if 0
3006	if (pVM->hm.s.fVirtApicRegs)
3007	{
3008	/* Enable APIC-register virtualization. */
3009	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3010	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3011
3012	/* Enable virtual-interrupt delivery. */
3013	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3014	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3015	}
3016	#endif
3017
3018	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3019	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3020	* done dynamically. */
3021	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3022	{
3023	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3024	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3025	AssertRCReturn(rc, rc);
3026	}
3027
3028	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3029	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3030	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3031	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3032	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3033
3034	/* Enable Pause-Loop exiting. */
3035	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3036	&& pVM->hm.s.vmx.cPleGapTicks
3037	&& pVM->hm.s.vmx.cPleWindowTicks)
3038	{
3039	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3040
3041	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3042	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3043	AssertRCReturn(rc, rc);
3044	}
3045
3046	if ((fVal & fZap) != fVal)
3047	{
3048	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3049	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3050	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3051	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3052	}
3053
3054	/* Commit it to the VMCS and update our cache. */
3055	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3056	AssertRCReturn(rc, rc);
3057	pVmcsInfo->u32ProcCtls2 = fVal;
3058
3059	return VINF_SUCCESS;
3060	}
3061
3062
3063	/**
3064	* Sets up processor-based VM-execution controls in the VMCS.
3065	*
3066	* @returns VBox status code.
3067	* @param pVCpu The cross context virtual CPU structure.
3068	* @param pVmcsInfo The VMCS info. object.
3069	*/
3070	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3071	{
3072	PVM pVM = pVCpu->CTX_SUFF(pVM);
3073
3074	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3075	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3076
3077	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3078	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3079	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3080	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3081	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3082	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3083	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3084
3085	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3086	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3087	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3088	{
3089	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3090	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3091	}
3092
3093	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3094	if (!pVM->hm.s.fNestedPaging)
3095	{
3096	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3097	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3098	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3099	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3100	}
3101
3102	/* Use TPR shadowing if supported by the CPU. */
3103	if ( PDMHasApic(pVM)
3104	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3105	{
3106	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3107	/* CR8 writes cause a VM-exit based on TPR threshold. */
3108	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3109	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3110	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3111	AssertRCReturn(rc, rc);
3112	}
3113	else
3114	{
3115	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3116	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3117	if (pVM->hm.s.fAllow64BitGuests)
3118	{
3119	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3120	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3121	}
3122	}
3123
3124	/* Use MSR-bitmaps if supported by the CPU. */
3125	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3126	{
3127	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3128	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3129	AssertRCReturn(rc, rc);
3130	}
3131
3132	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3133	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3134	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3135
3136	if ((fVal & fZap) != fVal)
3137	{
3138	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3139	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3140	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3141	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3142	}
3143
3144	/* Commit it to the VMCS and update our cache. */
3145	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3146	AssertRCReturn(rc, rc);
3147	pVmcsInfo->u32ProcCtls = fVal;
3148
3149	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3150	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3151	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3152
3153	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3154	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3155	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3156
3157	/* Sanity check, should not really happen. */
3158	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3159	{ /* likely */ }
3160	else
3161	{
3162	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3163	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3164	}
3165
3166	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3167	return VINF_SUCCESS;
3168	}
3169
3170
3171	/**
3172	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3173	* Processor-based VM-execution) control fields in the VMCS.
3174	*
3175	* @returns VBox status code.
3176	* @param pVCpu The cross context virtual CPU structure.
3177	* @param pVmcsInfo The VMCS info. object.
3178	*/
3179	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3180	{
3181	/* Set the auto-load/store MSR area addresses in the VMCS. */
3182	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3183	if (RT_SUCCESS(rc))
3184	{
3185	/* Set the VMCS link pointer in the VMCS. */
3186	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3187	if (RT_SUCCESS(rc))
3188	{
3189	/* Set the CR0/CR4 guest/host mask. */
3190	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3191	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3192	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3193	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3194	if (RT_SUCCESS(rc))
3195	{
3196	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3197	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3198	return VINF_SUCCESS;
3199	}
3200	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3201	}
3202	else
3203	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3204	}
3205	else
3206	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3207	return rc;
3208	}
3209
3210
3211	/**
3212	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3213	*
3214	* We shall setup those exception intercepts that don't change during the
3215	* lifetime of the VM here. The rest are done dynamically while loading the
3216	* guest state.
3217	*
3218	* @returns VBox status code.
3219	* @param pVCpu The cross context virtual CPU structure.
3220	* @param pVmcsInfo The VMCS info. object.
3221	*/
3222	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3223	{
3224	/*
3225	* The following exceptions are always intercepted:
3226	*
3227	* #AC - To prevent the guest from hanging the CPU.
3228	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3229	* recursive #DBs can cause a CPU hang.
3230	* #PF - To sync our shadow page tables when nested-paging is not used.
3231	*/
3232	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3233	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3234	\| RT_BIT(X86_XCPT_DB)
3235	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3236
3237	/* Commit it to the VMCS. */
3238	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3239	AssertRCReturn(rc, rc);
3240
3241	/* Update our cache of the exception bitmap. */
3242	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3243	return VINF_SUCCESS;
3244	}
3245
3246
3247	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3248	/**
3249	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3250	*
3251	* @returns VBox status code.
3252	* @param pVCpu The cross context virtual CPU structure.
3253	* @param pVmcsInfo The VMCS info. object.
3254	*/
3255	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3256	{
3257	PVM pVM = pVCpu->CTX_SUFF(pVM);
3258	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3259	if (RT_SUCCESS(rc))
3260	{
3261	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3262	if (RT_SUCCESS(rc))
3263	{
3264	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3265	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3266	if (RT_SUCCESS(rc))
3267	{
3268	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3269	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3270	if (RT_SUCCESS(rc))
3271	return VINF_SUCCESS;
3272
3273	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3274	}
3275	else
3276	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3277	}
3278	else
3279	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3280	}
3281	else
3282	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3283
3284	return rc;
3285	}
3286	#endif
3287
3288
3289	/**
3290	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3291	* VMX.
3292	*
3293	* @returns VBox status code.
3294	* @param pVCpu The cross context virtual CPU structure.
3295	* @param pVmcsInfo The VMCS info. object.
3296	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3297	*/
3298	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3299	{
3300	Assert(pVmcsInfo);
3301	Assert(pVmcsInfo->pvVmcs);
3302	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3303
3304	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3305	PVM pVM = pVCpu->CTX_SUFF(pVM);
3306	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3307	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3308
3309	LogFlowFunc(("\n"));
3310
3311	/*
3312	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3313	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3314	*/
3315	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3316	if (RT_SUCCESS(rc))
3317	{
3318	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3319	if (RT_SUCCESS(rc))
3320	{
3321	if (!fIsNstGstVmcs)
3322	{
3323	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3324	if (RT_SUCCESS(rc))
3325	{
3326	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3327	if (RT_SUCCESS(rc))
3328	{
3329	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3330	if (RT_SUCCESS(rc))
3331	{
3332	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3333	if (RT_SUCCESS(rc))
3334	{ /* likely */ }
3335	else
3336	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3337	}
3338	else
3339	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3340	}
3341	else
3342	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3343	}
3344	else
3345	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3346	}
3347	else
3348	{
3349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3350	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3351	if (RT_SUCCESS(rc))
3352	{ /* likely */ }
3353	else
3354	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3355	#else
3356	AssertFailed();
3357	#endif
3358	}
3359	}
3360	else
3361	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3362	}
3363	else
3364	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3365
3366	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3367	if (RT_SUCCESS(rc))
3368	{
3369	rc = hmR0VmxClearVmcs(pVmcsInfo);
3370	if (RT_SUCCESS(rc))
3371	{ /* likely */ }
3372	else
3373	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3374	}
3375
3376	/*
3377	* Update the last-error record both for failures and success, so we
3378	* can propagate the status code back to ring-3 for diagnostics.
3379	*/
3380	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3381	NOREF(pszVmcs);
3382	return rc;
3383	}
3384
3385
3386	/**
3387	* Does global VT-x initialization (called during module initialization).
3388	*
3389	* @returns VBox status code.
3390	*/
3391	VMMR0DECL(int) VMXR0GlobalInit(void)
3392	{
3393	#ifdef HMVMX_USE_FUNCTION_TABLE
3394	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3395	# ifdef VBOX_STRICT
3396	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3397	Assert(g_apfnVMExitHandlers[i]);
3398	# endif
3399	#endif
3400	return VINF_SUCCESS;
3401	}
3402
3403
3404	/**
3405	* Does global VT-x termination (called during module termination).
3406	*/
3407	VMMR0DECL(void) VMXR0GlobalTerm()
3408	{
3409	/* Nothing to do currently. */
3410	}
3411
3412
3413	/**
3414	* Sets up and activates VT-x on the current CPU.
3415	*
3416	* @returns VBox status code.
3417	* @param pHostCpu The HM physical-CPU structure.
3418	* @param pVM The cross context VM structure. Can be
3419	* NULL after a host resume operation.
3420	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3421	* fEnabledByHost is @c true).
3422	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3423	* @a fEnabledByHost is @c true).
3424	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3425	* enable VT-x on the host.
3426	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3427	*/
3428	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3429	PCSUPHWVIRTMSRS pHwvirtMsrs)
3430	{
3431	Assert(pHostCpu);
3432	Assert(pHwvirtMsrs);
3433	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3434
3435	/* Enable VT-x if it's not already enabled by the host. */
3436	if (!fEnabledByHost)
3437	{
3438	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3439	if (RT_FAILURE(rc))
3440	return rc;
3441	}
3442
3443	/*
3444	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3445	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3446	* invalidated when flushing by VPID.
3447	*/
3448	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3449	{
3450	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3451	pHostCpu->fFlushAsidBeforeUse = false;
3452	}
3453	else
3454	pHostCpu->fFlushAsidBeforeUse = true;
3455
3456	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3457	++pHostCpu->cTlbFlushes;
3458
3459	return VINF_SUCCESS;
3460	}
3461
3462
3463	/**
3464	* Deactivates VT-x on the current CPU.
3465	*
3466	* @returns VBox status code.
3467	* @param pvCpuPage Pointer to the VMXON region.
3468	* @param HCPhysCpuPage Physical address of the VMXON region.
3469	*
3470	* @remarks This function should never be called when SUPR0EnableVTx() or
3471	* similar was used to enable VT-x on the host.
3472	*/
3473	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3474	{
3475	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3476
3477	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3478	return hmR0VmxLeaveRootMode();
3479	}
3480
3481
3482	/**
3483	* Does per-VM VT-x initialization.
3484	*
3485	* @returns VBox status code.
3486	* @param pVM The cross context VM structure.
3487	*/
3488	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3489	{
3490	LogFlowFunc(("pVM=%p\n", pVM));
3491
3492	int rc = hmR0VmxStructsAlloc(pVM);
3493	if (RT_FAILURE(rc))
3494	{
3495	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3496	return rc;
3497	}
3498
3499	return VINF_SUCCESS;
3500	}
3501
3502
3503	/**
3504	* Does per-VM VT-x termination.
3505	*
3506	* @returns VBox status code.
3507	* @param pVM The cross context VM structure.
3508	*/
3509	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3510	{
3511	LogFlowFunc(("pVM=%p\n", pVM));
3512
3513	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3514	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3515	{
3516	Assert(pVM->hm.s.vmx.pvScratch);
3517	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3518	}
3519	#endif
3520	hmR0VmxStructsFree(pVM);
3521	return VINF_SUCCESS;
3522	}
3523
3524
3525	/**
3526	* Sets up the VM for execution using hardware-assisted VMX.
3527	* This function is only called once per-VM during initialization.
3528	*
3529	* @returns VBox status code.
3530	* @param pVM The cross context VM structure.
3531	*/
3532	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3533	{
3534	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3535	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3536
3537	LogFlowFunc(("pVM=%p\n", pVM));
3538
3539	/*
3540	* At least verify if VMX is enabled, since we can't check if we're in
3541	* VMX root mode or not without causing a #GP.
3542	*/
3543	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3544	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3545	{ /* likely */ }
3546	else
3547	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3548
3549	/*
3550	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3551	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3552	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3553	*/
3554	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3555	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3556	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3557	{
3558	LogRelFunc(("Invalid real-on-v86 state.\n"));
3559	return VERR_INTERNAL_ERROR;
3560	}
3561
3562	/* Initialize these always, see hmR3InitFinalizeR0().*/
3563	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3564	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3565
3566	/* Setup the tagged-TLB flush handlers. */
3567	int rc = hmR0VmxSetupTaggedTlb(pVM);
3568	if (RT_FAILURE(rc))
3569	{
3570	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3571	return rc;
3572	}
3573
3574	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3575	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3576	#if HC_ARCH_BITS == 64
3577	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3578	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3579	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3580	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3581	#endif
3582
3583	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3584	{
3585	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3586	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3587
3588	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3589	if (RT_SUCCESS(rc))
3590	{
3591	#if HC_ARCH_BITS == 32
3592	hmR0VmxInitVmcsReadCache(pVCpu);
3593	#endif
3594	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3595	if (pVM->cpum.ro.GuestFeatures.fVmx)
3596	{
3597	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3598	if (RT_SUCCESS(rc))
3599	{ /* likely */ }
3600	else
3601	{
3602	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3603	return rc;
3604	}
3605	}
3606	#endif
3607	}
3608	else
3609	{
3610	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3611	return rc;
3612	}
3613	}
3614
3615	return VINF_SUCCESS;
3616	}
3617
3618
3619	#if HC_ARCH_BITS == 32
3620	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3621	/**
3622	* Check if guest state allows safe use of 32-bit switcher again.
3623	*
3624	* Segment bases and protected mode structures must be 32-bit addressable
3625	* because the 32-bit switcher will ignore high dword when writing these VMCS
3626	* fields. See @bugref{8432} for details.
3627	*
3628	* @returns true if safe, false if must continue to use the 64-bit switcher.
3629	* @param pCtx Pointer to the guest-CPU context.
3630	*
3631	* @remarks No-long-jump zone!!!
3632	*/
3633	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3634	{
3635	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3636	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3637	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3638	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3639	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3640	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3641	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3642	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3643	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3644	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3645
3646	/* All good, bases are 32-bit. */
3647	return true;
3648	}
3649	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3650
3651	# ifdef VBOX_STRICT
3652	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3653	{
3654	switch (idxField)
3655	{
3656	case VMX_VMCS_GUEST_RIP:
3657	case VMX_VMCS_GUEST_RSP:
3658	case VMX_VMCS_GUEST_SYSENTER_EIP:
3659	case VMX_VMCS_GUEST_SYSENTER_ESP:
3660	case VMX_VMCS_GUEST_GDTR_BASE:
3661	case VMX_VMCS_GUEST_IDTR_BASE:
3662	case VMX_VMCS_GUEST_CS_BASE:
3663	case VMX_VMCS_GUEST_DS_BASE:
3664	case VMX_VMCS_GUEST_ES_BASE:
3665	case VMX_VMCS_GUEST_FS_BASE:
3666	case VMX_VMCS_GUEST_GS_BASE:
3667	case VMX_VMCS_GUEST_SS_BASE:
3668	case VMX_VMCS_GUEST_LDTR_BASE:
3669	case VMX_VMCS_GUEST_TR_BASE:
3670	case VMX_VMCS_GUEST_CR3:
3671	return true;
3672	}
3673	return false;
3674	}
3675
3676	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3677	{
3678	switch (idxField)
3679	{
3680	/* Read-only fields. */
3681	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3682	return true;
3683	}
3684	/* Remaining readable fields should also be writable. */
3685	return hmR0VmxIsValidWriteField(idxField);
3686	}
3687	# endif /* VBOX_STRICT */
3688
3689
3690	/**
3691	* Executes the specified handler in 64-bit mode.
3692	*
3693	* @returns VBox status code (no informational status codes).
3694	* @param pVCpu The cross context virtual CPU structure.
3695	* @param enmOp The operation to perform.
3696	* @param cParams Number of parameters.
3697	* @param paParam Array of 32-bit parameters.
3698	*/
3699	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3700	{
3701	PVM pVM = pVCpu->CTX_SUFF(pVM);
3702	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3703	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3704	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3705	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3706
3707	#ifdef VBOX_STRICT
3708	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3709	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3710
3711	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3712	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3713	#endif
3714
3715	/* Disable interrupts. */
3716	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3717
3718	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3719	RTCPUID idHostCpu = RTMpCpuId();
3720	CPUMR0SetLApic(pVCpu, idHostCpu);
3721	#endif
3722
3723	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3724
3725	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3726	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3727
3728	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3729	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3730	hmR0VmxClearVmcs(pVmcsInfo);
3731
3732	/* Leave VMX root mode and disable VMX. */
3733	VMXDisable();
3734	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3735
3736	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3737	CPUMSetHyperEIP(pVCpu, enmOp);
3738	for (int i = (int)cParams - 1; i >= 0; i--)
3739	CPUMPushHyper(pVCpu, paParam[i]);
3740
3741	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3742
3743	/* Call the switcher. */
3744	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3745	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3746
3747	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3748	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3749
3750	/* Re-enter VMX root mode. */
3751	int rc2 = VMXEnable(HCPhysCpuPage);
3752	if (RT_FAILURE(rc2))
3753	{
3754	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3755	ASMSetFlags(fOldEFlags);
3756	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3757	return rc2;
3758	}
3759
3760	/* Restore the VMCS as the current VMCS. */
3761	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3762	AssertRC(rc2);
3763	Assert(!(ASMGetFlags() & X86_EFL_IF));
3764	ASMSetFlags(fOldEFlags);
3765	return rc;
3766	}
3767
3768
3769	/**
3770	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3771	* supporting 64-bit guests.
3772	*
3773	* @returns VBox status code.
3774	* @param fResume Whether to VMLAUNCH or VMRESUME.
3775	* @param pCtx Pointer to the guest-CPU context.
3776	* @param pCache Pointer to the VMCS batch cache.
3777	* @param pVM The cross context VM structure.
3778	* @param pVCpu The cross context virtual CPU structure.
3779	*/
3780	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3781	{
3782	NOREF(fResume);
3783
3784	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3785	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3786	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3787
3788	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3789	pCache->uPos = 1;
3790	pCache->interPD = PGMGetInterPaeCR3(pVM);
3791	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3792	#endif
3793
3794	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3795	pCache->TestIn.HCPhysCpuPage = 0;
3796	pCache->TestIn.HCPhysVmcs = 0;
3797	pCache->TestIn.pCache = 0;
3798	pCache->TestOut.HCPhysVmcs = 0;
3799	pCache->TestOut.pCache = 0;
3800	pCache->TestOut.pCtx = 0;
3801	pCache->TestOut.eflags = 0;
3802	#else
3803	NOREF(pCache);
3804	#endif
3805
3806	uint32_t aParam[10];
3807	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3808	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3809	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3810	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3811	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3812	aParam[5] = 0;
3813	aParam[6] = VM_RC_ADDR(pVM, pVM);
3814	aParam[7] = 0;
3815	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3816	aParam[9] = 0;
3817
3818	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3819	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3820	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3821	#endif
3822	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3823
3824	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3825	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3826	Assert(pCtx->dr[4] == 10);
3827	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3828	#endif
3829
3830	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3831	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3832	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3833	pVmcsInfo->HCPhysVmcs));
3834	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3835	pCache->TestOut.HCPhysVmcs));
3836	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3837	pCache->TestOut.pCache));
3838	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3839	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3840	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3841	pCache->TestOut.pCtx));
3842	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3843	#endif
3844	NOREF(pCtx);
3845	return rc;
3846	}
3847	#endif
3848
3849
3850	/**
3851	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3852	* the VMCS.
3853	*
3854	* @returns VBox status code.
3855	*/
3856	static int hmR0VmxExportHostControlRegs(void)
3857	{
3858	RTCCUINTREG uReg = ASMGetCR0();
3859	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3860	AssertRCReturn(rc, rc);
3861
3862	uReg = ASMGetCR3();
3863	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3864	AssertRCReturn(rc, rc);
3865
3866	uReg = ASMGetCR4();
3867	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3868	AssertRCReturn(rc, rc);
3869	return rc;
3870	}
3871
3872
3873	/**
3874	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3875	* the host-state area in the VMCS.
3876	*
3877	* @returns VBox status code.
3878	* @param pVCpu The cross context virtual CPU structure.
3879	*/
3880	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3881	{
3882	#if HC_ARCH_BITS == 64
3883	/**
3884	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3885	* requirements. See hmR0VmxExportHostSegmentRegs().
3886	*/
3887	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3888	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3889	{ \
3890	bool fValidSelector = true; \
3891	if ((selValue) & X86_SEL_LDT) \
3892	{ \
3893	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3894	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3895	} \
3896	if (fValidSelector) \
3897	{ \
3898	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3899	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3900	} \
3901	(selValue) = 0; \
3902	}
3903
3904	/*
3905	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3906	* will be messed up. We should -not- save the messed up state without restoring
3907	* the original host-state, see @bugref{7240}.
3908	*
3909	* This apparently can happen (most likely the FPU changes), deal with it rather than
3910	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3911	*/
3912	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3913	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3914	{
3915	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3916	pVCpu->idCpu));
3917	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3918	}
3919	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3920	#else
3921	RT_NOREF(pVCpu);
3922	#endif
3923
3924	/*
3925	* Host DS, ES, FS and GS segment registers.
3926	*/
3927	#if HC_ARCH_BITS == 64
3928	RTSEL uSelDS = ASMGetDS();
3929	RTSEL uSelES = ASMGetES();
3930	RTSEL uSelFS = ASMGetFS();
3931	RTSEL uSelGS = ASMGetGS();
3932	#else
3933	RTSEL uSelDS = 0;
3934	RTSEL uSelES = 0;
3935	RTSEL uSelFS = 0;
3936	RTSEL uSelGS = 0;
3937	#endif
3938
3939	/*
3940	* Host CS and SS segment registers.
3941	*/
3942	RTSEL uSelCS = ASMGetCS();
3943	RTSEL uSelSS = ASMGetSS();
3944
3945	/*
3946	* Host TR segment register.
3947	*/
3948	RTSEL uSelTR = ASMGetTR();
3949
3950	#if HC_ARCH_BITS == 64
3951	/*
3952	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3953	* gain VM-entry and restore them before we get preempted.
3954	*
3955	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3956	*/
3957	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3958	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3959	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3960	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3961	# undef VMXLOCAL_ADJUST_HOST_SEG
3962	#endif
3963
3964	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3965	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3966	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3967	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3968	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3969	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3970	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3971	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3972	Assert(uSelCS);
3973	Assert(uSelTR);
3974
3975	/* Write these host selector fields into the host-state area in the VMCS. */
3976	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3977	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3978	#if HC_ARCH_BITS == 64
3979	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3980	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3981	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3982	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3983	#else
3984	NOREF(uSelDS);
3985	NOREF(uSelES);
3986	NOREF(uSelFS);
3987	NOREF(uSelGS);
3988	#endif
3989	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3990	AssertRCReturn(rc, rc);
3991
3992	/*
3993	* Host GDTR and IDTR.
3994	*/
3995	RTGDTR Gdtr;
3996	RTIDTR Idtr;
3997	RT_ZERO(Gdtr);
3998	RT_ZERO(Idtr);
3999	ASMGetGDTR(&Gdtr);
4000	ASMGetIDTR(&Idtr);
4001	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4002	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4003	AssertRCReturn(rc, rc);
4004
4005	#if HC_ARCH_BITS == 64
4006	/*
4007	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4008	* them to the maximum limit (0xffff) on every VM-exit.
4009	*/
4010	if (Gdtr.cbGdt != 0xffff)
4011	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4012
4013	/*
4014	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4015	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4016	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4017	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4018	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4019	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4020	* at 0xffff on hosts where we are sure it won't cause trouble.
4021	*/
4022	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4023	if (Idtr.cbIdt < 0x0fff)
4024	# else
4025	if (Idtr.cbIdt != 0xffff)
4026	# endif
4027	{
4028	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4029	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4030	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4031	}
4032	#endif
4033
4034	/*
4035	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4036	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4037	* RPL should be too in most cases.
4038	*/
4039	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4040	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4041
4042	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4043	#if HC_ARCH_BITS == 64
4044	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4045
4046	/*
4047	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4048	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4049	* restoration if the host has something else. Task switching is not supported in 64-bit
4050	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4051	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4052	*
4053	* [1] See Intel spec. 3.5 "System Descriptor Types".
4054	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4055	*/
4056	PVM pVM = pVCpu->CTX_SUFF(pVM);
4057	Assert(pDesc->System.u4Type == 11);
4058	if ( pDesc->System.u16LimitLow != 0x67
4059	\|\| pDesc->System.u4LimitHigh)
4060	{
4061	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4062	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4063	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4064	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4065	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4066	}
4067
4068	/*
4069	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4070	*/
4071	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4072	{
4073	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4074	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4075	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4076	{
4077	/* The GDT is read-only but the writable GDT is available. */
4078	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4079	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4080	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4081	AssertRCReturn(rc, rc);
4082	}
4083	}
4084	#else
4085	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4086	#endif
4087	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4088	AssertRCReturn(rc, rc);
4089
4090	/*
4091	* Host FS base and GS base.
4092	*/
4093	#if HC_ARCH_BITS == 64
4094	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4095	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4096	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4097	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4098	AssertRCReturn(rc, rc);
4099
4100	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4101	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4102	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4103	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4104	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4105	#endif
4106	return VINF_SUCCESS;
4107	}
4108
4109
4110	/**
4111	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4112	* host-state area of the VMCS.
4113	*
4114	* These MSRs will be automatically restored on the host after every successful
4115	* VM-exit.
4116	*
4117	* @returns VBox status code.
4118	* @param pVCpu The cross context virtual CPU structure.
4119	*
4120	* @remarks No-long-jump zone!!!
4121	*/
4122	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4123	{
4124	AssertPtr(pVCpu);
4125
4126	/*
4127	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4128	* rather than swapping them on every VM-entry.
4129	*/
4130	hmR0VmxLazySaveHostMsrs(pVCpu);
4131
4132	/*
4133	* Host Sysenter MSRs.
4134	*/
4135	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4136	#if HC_ARCH_BITS == 32
4137	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4138	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4139	#else
4140	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4141	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4142	#endif
4143	AssertRCReturn(rc, rc);
4144
4145	/*
4146	* Host EFER MSR.
4147	*
4148	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4149	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4150	*/
4151	PVM pVM = pVCpu->CTX_SUFF(pVM);
4152	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4153	{
4154	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4155	AssertRCReturn(rc, rc);
4156	}
4157
4158	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4159	* hmR0VmxExportGuestEntryExitCtls(). */
4160
4161	return VINF_SUCCESS;
4162	}
4163
4164
4165	/**
4166	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4167	*
4168	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4169	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4170	*
4171	* @returns true if we need to load guest EFER, false otherwise.
4172	* @param pVCpu The cross context virtual CPU structure.
4173	*
4174	* @remarks Requires EFER, CR4.
4175	* @remarks No-long-jump zone!!!
4176	*/
4177	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4178	{
4179	#ifdef HMVMX_ALWAYS_SWAP_EFER
4180	RT_NOREF(pVCpu);
4181	return true;
4182	#else
4183	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4184	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4185	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4186	if (CPUMIsGuestInLongModeEx(pCtx))
4187	return false;
4188	#endif
4189
4190	PVM pVM = pVCpu->CTX_SUFF(pVM);
4191	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4192	uint64_t const u64GuestEfer = pCtx->msrEFER;
4193
4194	/*
4195	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4196	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4197	*/
4198	if ( CPUMIsGuestInLongModeEx(pCtx)
4199	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4200	return true;
4201
4202	/*
4203	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4204	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4205	*
4206	* See Intel spec. 4.5 "IA-32e Paging".
4207	* See Intel spec. 4.1.1 "Three Paging Modes".
4208	*
4209	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4210	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4211	*/
4212	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4213	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4214	if ( (pCtx->cr4 & X86_CR4_PAE)
4215	&& (pCtx->cr0 & X86_CR0_PG)
4216	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4217	{
4218	/* Assert that host is NX capable. */
4219	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4220	return true;
4221	}
4222
4223	return false;
4224	#endif
4225	}
4226
4227	/**
4228	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4229	* VMCS.
4230	*
4231	* This is typically required when the guest changes paging mode.
4232	*
4233	* @returns VBox status code.
4234	* @param pVCpu The cross context virtual CPU structure.
4235	* @param pVmxTransient The VMX-transient structure.
4236	*
4237	* @remarks Requires EFER.
4238	* @remarks No-long-jump zone!!!
4239	*/
4240	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4241	{
4242	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4243	{
4244	PVM pVM = pVCpu->CTX_SUFF(pVM);
4245	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4246
4247	/*
4248	* VM-entry controls.
4249	*/
4250	{
4251	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4252	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4253
4254	/*
4255	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4256	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4257	*
4258	* For nested-guests, this is a mandatory VM-entry control. It's also
4259	* required because we do not want to leak host bits to the nested-guest.
4260	*/
4261	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4262
4263	/*
4264	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4265	*
4266	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4267	* required to get the nested-guest working with hardware-assisted VMX execution.
4268	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4269	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4270	* here rather than while merging the guest VMCS controls.
4271	*/
4272	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4273	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4274	else
4275	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4276
4277	/*
4278	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4279	*
4280	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4281	* regardless of whether the nested-guest VMCS specifies it because we are free to
4282	* load whatever MSRs we require and we do not need to modify the guest visible copy
4283	* of the VM-entry MSR load area.
4284	*/
4285	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4286	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4287	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4288	else
4289	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4290
4291	/*
4292	* The following should -not- be set (since we're not in SMM mode):
4293	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4294	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4295	*/
4296
4297	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4298	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4299
4300	if ((fVal & fZap) == fVal)
4301	{ /* likely */ }
4302	else
4303	{
4304	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4305	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4306	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4307	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4308	}
4309
4310	/* Commit it to the VMCS. */
4311	if (pVmcsInfo->u32EntryCtls != fVal)
4312	{
4313	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4314	AssertRCReturn(rc, rc);
4315	pVmcsInfo->u32EntryCtls = fVal;
4316	}
4317	}
4318
4319	/*
4320	* VM-exit controls.
4321	*/
4322	{
4323	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4324	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4325
4326	/*
4327	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4328	* supported the 1-setting of this bit.
4329	*
4330	* For nested-guests, we set the "save debug controls" as the converse
4331	* "load debug controls" is mandatory for nested-guests anyway.
4332	*/
4333	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4334
4335	/*
4336	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4337	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4338	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4339	* hmR0VmxExportHostMsrs().
4340	*
4341	* For nested-guests, we always set this bit as we do not support 32-bit
4342	* hosts.
4343	*/
4344	#if HC_ARCH_BITS == 64
4345	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4346	#else
4347	Assert(!pVmxTransient->fIsNestedGuest);
4348	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4349	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4350	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4351	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4352	{
4353	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4354	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4355	}
4356	else
4357	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4358	#endif
4359
4360	/*
4361	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4362	*
4363	* For nested-guests, we should use the "save IA32_EFER" control if we also
4364	* used the "load IA32_EFER" control while exporting VM-entry controls.
4365	*/
4366	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4367	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4368	{
4369	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4370	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4371	}
4372
4373	/*
4374	* Enable saving of the VMX-preemption timer value on VM-exit.
4375	* For nested-guests, currently not exposed/used.
4376	*/
4377	if ( pVM->hm.s.vmx.fUsePreemptTimer
4378	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4379	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4380
4381	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4382	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4383
4384	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4385	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4386	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4387
4388	if ((fVal & fZap) == fVal)
4389	{ /* likely */ }
4390	else
4391	{
4392	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4393	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4394	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4395	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4396	}
4397
4398	/* Commit it to the VMCS. */
4399	if (pVmcsInfo->u32ExitCtls != fVal)
4400	{
4401	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4402	AssertRCReturn(rc, rc);
4403	pVmcsInfo->u32ExitCtls = fVal;
4404	}
4405	}
4406
4407	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4408	}
4409	return VINF_SUCCESS;
4410	}
4411
4412
4413	/**
4414	* Sets the TPR threshold in the VMCS.
4415	*
4416	* @returns VBox status code.
4417	* @param pVCpu The cross context virtual CPU structure.
4418	* @param pVmcsInfo The VMCS info. object.
4419	* @param u32TprThreshold The TPR threshold (task-priority class only).
4420	*/
4421	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4422	{
4423	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4424	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4425	RT_NOREF2(pVCpu, pVmcsInfo);
4426	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4427	}
4428
4429
4430	/**
4431	* Exports the guest APIC TPR state into the VMCS.
4432	*
4433	* @returns VBox status code.
4434	* @param pVCpu The cross context virtual CPU structure.
4435	* @param pVmxTransient The VMX-transient structure.
4436	*
4437	* @remarks No-long-jump zone!!!
4438	*/
4439	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4440	{
4441	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4442	{
4443	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4444
4445	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4446	if (!pVmxTransient->fIsNestedGuest)
4447	{
4448	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4449	&& APICIsEnabled(pVCpu))
4450	{
4451	/*
4452	* Setup TPR shadowing.
4453	*/
4454	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4455	{
4456	bool fPendingIntr = false;
4457	uint8_t u8Tpr = 0;
4458	uint8_t u8PendingIntr = 0;
4459	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4460	AssertRCReturn(rc, rc);
4461
4462	/*
4463	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4464	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4465	* priority of the pending interrupt so we can deliver the interrupt. If there
4466	* are no interrupts pending, set threshold to 0 to not cause any
4467	* TPR-below-threshold VM-exits.
4468	*/
4469	Assert(pVmcsInfo->pbVirtApic);
4470	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4471	uint32_t u32TprThreshold = 0;
4472	if (fPendingIntr)
4473	{
4474	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4475	(which is the Task-Priority Class). */
4476	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4477	const uint8_t u8TprPriority = u8Tpr >> 4;
4478	if (u8PendingPriority <= u8TprPriority)
4479	u32TprThreshold = u8PendingPriority;
4480	}
4481
4482	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4483	AssertRCReturn(rc, rc);
4484	}
4485	}
4486	}
4487	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4488	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4489	}
4490	return VINF_SUCCESS;
4491	}
4492
4493
4494	/**
4495	* Gets the guest interruptibility-state.
4496	*
4497	* @returns Guest's interruptibility-state.
4498	* @param pVCpu The cross context virtual CPU structure.
4499	* @param pVmcsInfo The VMCS info. object.
4500	*
4501	* @remarks No-long-jump zone!!!
4502	*/
4503	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4504	{
4505	/*
4506	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4507	*/
4508	uint32_t fIntrState = 0;
4509	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4510	{
4511	/* If inhibition is active, RIP and RFLAGS should've been updated
4512	(i.e. read previously from the VMCS or from ring-3). */
4513	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4514	#ifdef VBOX_STRICT
4515	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4516	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4517	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4518	#endif
4519	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4520	{
4521	if (pCtx->eflags.Bits.u1IF)
4522	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4523	else
4524	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4525	}
4526	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4527	{
4528	/*
4529	* We can clear the inhibit force flag as even if we go back to the recompiler
4530	* without executing guest code in VT-x, the flag's condition to be cleared is
4531	* met and thus the cleared state is correct.
4532	*/
4533	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4534	}
4535	}
4536
4537	/*
4538	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4539	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4540	* setting this would block host-NMIs and IRET will not clear the blocking.
4541	*
4542	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4543	*
4544	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4545	*/
4546	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4547	&& CPUMIsGuestNmiBlocking(pVCpu))
4548	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4549
4550	return fIntrState;
4551	}
4552
4553
4554	/**
4555	* Exports the exception intercepts required for guest execution in the VMCS.
4556	*
4557	* @returns VBox status code.
4558	* @param pVCpu The cross context virtual CPU structure.
4559	* @param pVmxTransient The VMX-transient structure.
4560	*
4561	* @remarks No-long-jump zone!!!
4562	*/
4563	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4564	{
4565	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4566	{
4567	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4568	if ( !pVmxTransient->fIsNestedGuest
4569	&& pVCpu->hm.s.fGIMTrapXcptUD)
4570	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4571	else
4572	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4573
4574	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4575	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4576	}
4577	return VINF_SUCCESS;
4578	}
4579
4580
4581	/**
4582	* Exports the guest's RIP into the guest-state area in the VMCS.
4583	*
4584	* @returns VBox status code.
4585	* @param pVCpu The cross context virtual CPU structure.
4586	*
4587	* @remarks No-long-jump zone!!!
4588	*/
4589	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4590	{
4591	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4592	{
4593	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4594
4595	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4596	AssertRCReturn(rc, rc);
4597
4598	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4599	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4600	}
4601	return VINF_SUCCESS;
4602	}
4603
4604
4605	/**
4606	* Exports the guest's RSP into the guest-state area in the VMCS.
4607	*
4608	* @returns VBox status code.
4609	* @param pVCpu The cross context virtual CPU structure.
4610	*
4611	* @remarks No-long-jump zone!!!
4612	*/
4613	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4614	{
4615	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4616	{
4617	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4618
4619	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4620	AssertRCReturn(rc, rc);
4621
4622	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4623	}
4624	return VINF_SUCCESS;
4625	}
4626
4627
4628	/**
4629	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4630	*
4631	* @returns VBox status code.
4632	* @param pVCpu The cross context virtual CPU structure.
4633	* @param pVmxTransient The VMX-transient structure.
4634	*
4635	* @remarks No-long-jump zone!!!
4636	*/
4637	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4638	{
4639	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4640	{
4641	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4642
4643	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4644	Let us assert it as such and use 32-bit VMWRITE. */
4645	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4646	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4647	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4648	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4649
4650	/*
4651	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4652	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4653	* can run the real-mode guest code under Virtual 8086 mode.
4654	*/
4655	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4656	if (pVmcsInfo->RealMode.fRealOnV86Active)
4657	{
4658	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4659	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4660	Assert(!pVmxTransient->fIsNestedGuest);
4661	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4662	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4663	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4664	}
4665
4666	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4667	AssertRCReturn(rc, rc);
4668
4669	/*
4670	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4671	*
4672	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4673	* through the hypervisor debugger using EFLAGS.TF.
4674	*/
4675	if ( !pVmxTransient->fIsNestedGuest
4676	&& !pVCpu->hm.s.fSingleInstruction
4677	&& fEFlags.Bits.u1TF)
4678	{
4679	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4680	* premature trips to ring-3 esp since IEM does not yet handle it. */
4681	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4682	AssertRCReturn(rc, rc);
4683	}
4684	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4685	* nested-guest VMCS. */
4686
4687	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4688	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4689	}
4690	return VINF_SUCCESS;
4691	}
4692
4693
4694	/**
4695	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4696	*
4697	* The guest FPU state is always pre-loaded hence we don't need to bother about
4698	* sharing FPU related CR0 bits between the guest and host.
4699	*
4700	* @returns VBox status code.
4701	* @param pVCpu The cross context virtual CPU structure.
4702	* @param pVmxTransient The VMX-transient structure.
4703	*
4704	* @remarks No-long-jump zone!!!
4705	*/
4706	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4707	{
4708	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4709	{
4710	PVM pVM = pVCpu->CTX_SUFF(pVM);
4711	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4712
4713	/*
4714	* Figure out fixed CR0 bits in VMX operation.
4715	*/
4716	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4717	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4718	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4719	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4720	else
4721	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4722
4723	if (!pVmxTransient->fIsNestedGuest)
4724	{
4725	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4726	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4727	uint64_t const u64ShadowCr0 = u64GuestCr0;
4728	Assert(!RT_HI_U32(u64GuestCr0));
4729
4730	/*
4731	* Setup VT-x's view of the guest CR0.
4732	*/
4733	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4734	if (pVM->hm.s.fNestedPaging)
4735	{
4736	if (CPUMIsGuestPagingEnabled(pVCpu))
4737	{
4738	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4739	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4740	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4741	}
4742	else
4743	{
4744	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4745	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4746	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4747	}
4748
4749	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4750	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4751	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4752	}
4753	else
4754	{
4755	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4756	u64GuestCr0 \|= X86_CR0_WP;
4757	}
4758
4759	/*
4760	* Guest FPU bits.
4761	*
4762	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4763	* using CR0.TS.
4764	*
4765	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4766	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4767	*/
4768	u64GuestCr0 \|= X86_CR0_NE;
4769
4770	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4771	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4772
4773	/*
4774	* Update exception intercepts.
4775	*/
4776	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4777	if (pVmcsInfo->RealMode.fRealOnV86Active)
4778	{
4779	Assert(PDMVmmDevHeapIsEnabled(pVM));
4780	Assert(pVM->hm.s.vmx.pRealModeTSS);
4781	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4782	}
4783	else
4784	{
4785	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4786	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4787	if (fInterceptMF)
4788	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4789	}
4790
4791	/* Additional intercepts for debugging, define these yourself explicitly. */
4792	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4793	uXcptBitmap \|= 0
4794	\| RT_BIT(X86_XCPT_BP)
4795	\| RT_BIT(X86_XCPT_DE)
4796	\| RT_BIT(X86_XCPT_NM)
4797	\| RT_BIT(X86_XCPT_TS)
4798	\| RT_BIT(X86_XCPT_UD)
4799	\| RT_BIT(X86_XCPT_NP)
4800	\| RT_BIT(X86_XCPT_SS)
4801	\| RT_BIT(X86_XCPT_GP)
4802	\| RT_BIT(X86_XCPT_PF)
4803	\| RT_BIT(X86_XCPT_MF)
4804	;
4805	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4806	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4807	#endif
4808	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4809	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4810	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4811
4812	/* Apply the fixed CR0 bits and enable caching. */
4813	u64GuestCr0 \|= fSetCr0;
4814	u64GuestCr0 &= fZapCr0;
4815	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4816
4817	/* Commit the CR0 and related fields to the guest VMCS. */
4818	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4819	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4820	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4821	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4822	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4823	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4824	AssertRCReturn(rc, rc);
4825
4826	/* Update our caches. */
4827	pVmcsInfo->u32ProcCtls = uProcCtls;
4828	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4829
4830	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4831	}
4832	else
4833	{
4834	/*
4835	* With nested-guests, we may have extended the guest/host mask here (since we
4836	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
4837	* mask can include more bits (to read from the nested-guest CR0 read-shadow) than
4838	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
4839	* those bits from the nested-guest CR0 into the nested-guest CR0 read shadow.
4840	*/
4841	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4842	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4843	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx);
4844	Assert(!RT_HI_U32(u64GuestCr0));
4845	Assert(u64GuestCr0 & X86_CR0_NE);
4846
4847	/* Apply the fixed CR0 bits and enable caching. */
4848	u64GuestCr0 \|= fSetCr0;
4849	u64GuestCr0 &= fZapCr0;
4850	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4851
4852	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4853	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4854	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4855	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4856	AssertRCReturn(rc, rc);
4857
4858	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4859	}
4860
4861	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4862	}
4863
4864	return VINF_SUCCESS;
4865	}
4866
4867
4868	/**
4869	* Exports the guest control registers (CR3, CR4) into the guest-state area
4870	* in the VMCS.
4871	*
4872	* @returns VBox strict status code.
4873	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4874	* without unrestricted guest access and the VMMDev is not presently
4875	* mapped (e.g. EFI32).
4876	*
4877	* @param pVCpu The cross context virtual CPU structure.
4878	* @param pVmxTransient The VMX-transient structure.
4879	*
4880	* @remarks No-long-jump zone!!!
4881	*/
4882	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4883	{
4884	int rc = VINF_SUCCESS;
4885	PVM pVM = pVCpu->CTX_SUFF(pVM);
4886
4887	/*
4888	* Guest CR2.
4889	* It's always loaded in the assembler code. Nothing to do here.
4890	*/
4891
4892	/*
4893	* Guest CR3.
4894	*/
4895	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4896	{
4897	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4898
4899	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4900	if (pVM->hm.s.fNestedPaging)
4901	{
4902	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4903	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4904
4905	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4906	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4907	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4908	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4909
4910	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4911	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4912	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4913
4914	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4915	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4916	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4917	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4918	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4919	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4920	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4921
4922	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4923	AssertRCReturn(rc, rc);
4924
4925	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4926	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4927	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4928	{
4929	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4930	if (CPUMIsGuestInPAEModeEx(pCtx))
4931	{
4932	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4933	AssertRCReturn(rc, rc);
4934	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4935	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4936	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4937	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4938	AssertRCReturn(rc, rc);
4939	}
4940
4941	/*
4942	* The guest's view of its CR3 is unblemished with nested paging when the
4943	* guest is using paging or we have unrestricted guest execution to handle
4944	* the guest when it's not using paging.
4945	*/
4946	GCPhysGuestCR3 = pCtx->cr3;
4947	}
4948	else
4949	{
4950	/*
4951	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4952	* thinks it accesses physical memory directly, we use our identity-mapped
4953	* page table to map guest-linear to guest-physical addresses. EPT takes care
4954	* of translating it to host-physical addresses.
4955	*/
4956	RTGCPHYS GCPhys;
4957	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4958
4959	/* We obtain it here every time as the guest could have relocated this PCI region. */
4960	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4961	if (RT_SUCCESS(rc))
4962	{ /* likely */ }
4963	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4964	{
4965	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4966	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4967	}
4968	else
4969	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4970
4971	GCPhysGuestCR3 = GCPhys;
4972	}
4973
4974	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4975	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4976	AssertRCReturn(rc, rc);
4977	}
4978	else
4979	{
4980	/* Non-nested paging case, just use the hypervisor's CR3. */
4981	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4982
4983	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4984	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4985	AssertRCReturn(rc, rc);
4986	}
4987
4988	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4989	}
4990
4991	/*
4992	* Guest CR4.
4993	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4994	*/
4995	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4996	{
4997	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4998	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4999
5000	/*
5001	* Figure out fixed CR4 bits in VMX operation.
5002	*/
5003	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5004	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5005
5006	/*
5007	* With nested-guests, we may have extended the guest/host mask here (since we
5008	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5009	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5010	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5011	* those bits from the nested-guest CR4 into the nested-guest CR4 read shadow.
5012	*/
5013	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5014	uint64_t u64GuestCr4 = pCtx->cr4;
5015	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5016	? pCtx->cr4
5017	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx);
5018	Assert(!RT_HI_U32(u64GuestCr4));
5019
5020	/*
5021	* Setup VT-x's view of the guest CR4.
5022	*
5023	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5024	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5025	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5026	*
5027	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5028	*/
5029	if (pVmcsInfo->RealMode.fRealOnV86Active)
5030	{
5031	Assert(pVM->hm.s.vmx.pRealModeTSS);
5032	Assert(PDMVmmDevHeapIsEnabled(pVM));
5033	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5034	}
5035
5036	if (pVM->hm.s.fNestedPaging)
5037	{
5038	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5039	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5040	{
5041	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5042	u64GuestCr4 \|= X86_CR4_PSE;
5043	/* Our identity mapping is a 32-bit page directory. */
5044	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5045	}
5046	/* else use guest CR4.*/
5047	}
5048	else
5049	{
5050	Assert(!pVmxTransient->fIsNestedGuest);
5051
5052	/*
5053	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5054	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5055	*/
5056	switch (pVCpu->hm.s.enmShadowMode)
5057	{
5058	case PGMMODE_REAL: /* Real-mode. */
5059	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5060	case PGMMODE_32_BIT: /* 32-bit paging. */
5061	{
5062	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5063	break;
5064	}
5065
5066	case PGMMODE_PAE: /* PAE paging. */
5067	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5068	{
5069	u64GuestCr4 \|= X86_CR4_PAE;
5070	break;
5071	}
5072
5073	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5074	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5075	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5076	break;
5077	#endif
5078	default:
5079	AssertFailed();
5080	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5081	}
5082	}
5083
5084	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5085	u64GuestCr4 \|= fSetCr4;
5086	u64GuestCr4 &= fZapCr4;
5087
5088	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5089	/** @todo Fix to 64-bit when we drop 32-bit. */
5090	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5091	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5092	AssertRCReturn(rc, rc);
5093
5094	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5095	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5096
5097	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5098
5099	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5100	}
5101	return rc;
5102	}
5103
5104
5105	/**
5106	* Exports the guest debug registers into the guest-state area in the VMCS.
5107	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5108	*
5109	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5110	*
5111	* @returns VBox status code.
5112	* @param pVCpu The cross context virtual CPU structure.
5113	* @param pVmxTransient The VMX-transient structure.
5114	*
5115	* @remarks No-long-jump zone!!!
5116	*/
5117	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5118	{
5119	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5120
5121	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5122	* stepping. */
5123	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5124	if (pVmxTransient->fIsNestedGuest)
5125	{
5126	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5127	AssertRCReturn(rc, rc);
5128	return VINF_SUCCESS;
5129	}
5130
5131	#ifdef VBOX_STRICT
5132	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5133	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5134	{
5135	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5136	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5137	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5138	}
5139	#endif
5140
5141	bool fSteppingDB = false;
5142	bool fInterceptMovDRx = false;
5143	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5144	if (pVCpu->hm.s.fSingleInstruction)
5145	{
5146	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5147	PVM pVM = pVCpu->CTX_SUFF(pVM);
5148	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5149	{
5150	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5151	Assert(fSteppingDB == false);
5152	}
5153	else
5154	{
5155	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5156	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5157	pVCpu->hm.s.fClearTrapFlag = true;
5158	fSteppingDB = true;
5159	}
5160	}
5161
5162	uint32_t u32GuestDr7;
5163	if ( fSteppingDB
5164	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5165	{
5166	/*
5167	* Use the combined guest and host DRx values found in the hypervisor register set
5168	* because the hypervisor debugger has breakpoints active or someone is single stepping
5169	* on the host side without a monitor trap flag.
5170	*
5171	* Note! DBGF expects a clean DR6 state before executing guest code.
5172	*/
5173	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5174	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5175	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5176	{
5177	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5178	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5179	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5180	}
5181	else
5182	#endif
5183	if (!CPUMIsHyperDebugStateActive(pVCpu))
5184	{
5185	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5186	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5187	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5188	}
5189
5190	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5191	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5192	pVCpu->hm.s.fUsingHyperDR7 = true;
5193	fInterceptMovDRx = true;
5194	}
5195	else
5196	{
5197	/*
5198	* If the guest has enabled debug registers, we need to load them prior to
5199	* executing guest code so they'll trigger at the right time.
5200	*/
5201	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5202	{
5203	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5204	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5205	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5206	{
5207	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5208	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5209	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5210	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5211	}
5212	else
5213	#endif
5214	if (!CPUMIsGuestDebugStateActive(pVCpu))
5215	{
5216	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5217	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5218	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5219	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5220	}
5221	Assert(!fInterceptMovDRx);
5222	}
5223	/*
5224	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5225	* must intercept #DB in order to maintain a correct DR6 guest value, and
5226	* because we need to intercept it to prevent nested #DBs from hanging the
5227	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5228	*/
5229	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5230	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5231	&& !CPUMIsGuestDebugStateActive(pVCpu))
5232	#else
5233	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5234	#endif
5235	{
5236	fInterceptMovDRx = true;
5237	}
5238
5239	/* Update DR7 with the actual guest value. */
5240	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5241	pVCpu->hm.s.fUsingHyperDR7 = false;
5242	}
5243
5244	if (fInterceptMovDRx)
5245	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5246	else
5247	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5248
5249	/*
5250	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5251	* monitor-trap flag and update our cache.
5252	*/
5253	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5254	{
5255	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5256	AssertRCReturn(rc2, rc2);
5257	pVmcsInfo->u32ProcCtls = uProcCtls;
5258	}
5259
5260	/*
5261	* Update guest DR7.
5262	*/
5263	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5264	AssertRCReturn(rc, rc);
5265
5266	/*
5267	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5268	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5269	*
5270	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5271	*/
5272	if (fSteppingDB)
5273	{
5274	Assert(pVCpu->hm.s.fSingleInstruction);
5275	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5276
5277	uint32_t fIntrState = 0;
5278	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5279	AssertRCReturn(rc, rc);
5280
5281	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5282	{
5283	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5284	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5285	AssertRCReturn(rc, rc);
5286	}
5287	}
5288
5289	return VINF_SUCCESS;
5290	}
5291
5292
5293	#ifdef VBOX_STRICT
5294	/**
5295	* Strict function to validate segment registers.
5296	*
5297	* @param pVCpu The cross context virtual CPU structure.
5298	* @param pVmcsInfo The VMCS info. object.
5299	*
5300	* @remarks Will import guest CR0 on strict builds during validation of
5301	* segments.
5302	*/
5303	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5304	{
5305	/*
5306	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5307	*
5308	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5309	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5310	* unusable bit and doesn't change the guest-context value.
5311	*/
5312	PVM pVM = pVCpu->CTX_SUFF(pVM);
5313	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5314	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5315	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5316	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5317	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5318	{
5319	/* Protected mode checks */
5320	/* CS */
5321	Assert(pCtx->cs.Attr.n.u1Present);
5322	Assert(!(pCtx->cs.Attr.u & 0xf00));
5323	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5324	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5325	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5326	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5327	\|\| (pCtx->cs.Attr.n.u1Granularity));
5328	/* CS cannot be loaded with NULL in protected mode. */
5329	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5330	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5331	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5332	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5333	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5334	else
5335	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5336	/* SS */
5337	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5338	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5339	if ( !(pCtx->cr0 & X86_CR0_PE)
5340	\|\| pCtx->cs.Attr.n.u4Type == 3)
5341	{
5342	Assert(!pCtx->ss.Attr.n.u2Dpl);
5343	}
5344	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5345	{
5346	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5347	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5348	Assert(pCtx->ss.Attr.n.u1Present);
5349	Assert(!(pCtx->ss.Attr.u & 0xf00));
5350	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5351	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5352	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5353	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5354	\|\| (pCtx->ss.Attr.n.u1Granularity));
5355	}
5356	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5357	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5358	{
5359	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5360	Assert(pCtx->ds.Attr.n.u1Present);
5361	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5362	Assert(!(pCtx->ds.Attr.u & 0xf00));
5363	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5364	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5365	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5366	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5367	\|\| (pCtx->ds.Attr.n.u1Granularity));
5368	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5369	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5370	}
5371	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5372	{
5373	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5374	Assert(pCtx->es.Attr.n.u1Present);
5375	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5376	Assert(!(pCtx->es.Attr.u & 0xf00));
5377	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5378	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5379	\|\| !(pCtx->es.Attr.n.u1Granularity));
5380	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5381	\|\| (pCtx->es.Attr.n.u1Granularity));
5382	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5383	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5384	}
5385	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5386	{
5387	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5388	Assert(pCtx->fs.Attr.n.u1Present);
5389	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5390	Assert(!(pCtx->fs.Attr.u & 0xf00));
5391	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5392	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5393	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5394	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5395	\|\| (pCtx->fs.Attr.n.u1Granularity));
5396	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5397	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5398	}
5399	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5400	{
5401	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5402	Assert(pCtx->gs.Attr.n.u1Present);
5403	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5404	Assert(!(pCtx->gs.Attr.u & 0xf00));
5405	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5406	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5407	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5408	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5409	\|\| (pCtx->gs.Attr.n.u1Granularity));
5410	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5411	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5412	}
5413	/* 64-bit capable CPUs. */
5414	# if HC_ARCH_BITS == 64
5415	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5416	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5417	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5418	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5419	# endif
5420	}
5421	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5422	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5423	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5424	{
5425	/* Real and v86 mode checks. */
5426	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5427	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5428	if (pVmcsInfo->RealMode.fRealOnV86Active)
5429	{
5430	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5431	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5432	}
5433	else
5434	{
5435	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5436	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5437	}
5438
5439	/* CS */
5440	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5441	Assert(pCtx->cs.u32Limit == 0xffff);
5442	Assert(u32CSAttr == 0xf3);
5443	/* SS */
5444	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5445	Assert(pCtx->ss.u32Limit == 0xffff);
5446	Assert(u32SSAttr == 0xf3);
5447	/* DS */
5448	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5449	Assert(pCtx->ds.u32Limit == 0xffff);
5450	Assert(u32DSAttr == 0xf3);
5451	/* ES */
5452	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5453	Assert(pCtx->es.u32Limit == 0xffff);
5454	Assert(u32ESAttr == 0xf3);
5455	/* FS */
5456	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5457	Assert(pCtx->fs.u32Limit == 0xffff);
5458	Assert(u32FSAttr == 0xf3);
5459	/* GS */
5460	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5461	Assert(pCtx->gs.u32Limit == 0xffff);
5462	Assert(u32GSAttr == 0xf3);
5463	/* 64-bit capable CPUs. */
5464	# if HC_ARCH_BITS == 64
5465	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5466	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5467	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5468	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5469	# endif
5470	}
5471	}
5472	#endif /* VBOX_STRICT */
5473
5474
5475	/**
5476	* Exports a guest segment register into the guest-state area in the VMCS.
5477	*
5478	* @returns VBox status code.
5479	* @param pVCpu The cross context virtual CPU structure.
5480	* @param pVmcsInfo The VMCS info. object.
5481	* @param iSegReg The segment register number (X86_SREG_XXX).
5482	* @param pSelReg Pointer to the segment selector.
5483	*
5484	* @remarks No-long-jump zone!!!
5485	*/
5486	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5487	{
5488	Assert(iSegReg < X86_SREG_COUNT);
5489	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5490	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5491	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5492	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5493
5494	uint32_t u32Access = pSelReg->Attr.u;
5495	if (pVmcsInfo->RealMode.fRealOnV86Active)
5496	{
5497	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5498	u32Access = 0xf3;
5499	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5500	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5501	RT_NOREF_PV(pVCpu);
5502	}
5503	else
5504	{
5505	/*
5506	* The way to differentiate between whether this is really a null selector or was just
5507	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5508	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5509	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5510	* NULL selectors loaded in protected-mode have their attribute as 0.
5511	*/
5512	if (!u32Access)
5513	u32Access = X86DESCATTR_UNUSABLE;
5514	}
5515
5516	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5517	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5518	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5519
5520	/*
5521	* Commit it to the VMCS.
5522	*/
5523	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5524	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5525	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5526	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5527	AssertRCReturn(rc, rc);
5528	return rc;
5529	}
5530
5531
5532	/**
5533	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5534	* area in the VMCS.
5535	*
5536	* @returns VBox status code.
5537	* @param pVCpu The cross context virtual CPU structure.
5538	* @param pVmxTransient The VMX-transient structure.
5539	*
5540	* @remarks Will import guest CR0 on strict builds during validation of
5541	* segments.
5542	* @remarks No-long-jump zone!!!
5543	*/
5544	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5545	{
5546	int rc = VERR_INTERNAL_ERROR_5;
5547	PVM pVM = pVCpu->CTX_SUFF(pVM);
5548	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5549	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5550
5551	/*
5552	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5553	*/
5554	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5555	{
5556	#ifdef VBOX_WITH_REM
5557	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5558	{
5559	Assert(!pVmxTransient->fIsNestedGuest);
5560	Assert(pVM->hm.s.vmx.pRealModeTSS);
5561	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5562	if ( pVmcsInfo->fWasInRealMode
5563	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5564	{
5565	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5566	in real-mode (e.g. OpenBSD 4.0) */
5567	REMFlushTBs(pVM);
5568	Log4Func(("Switch to protected mode detected!\n"));
5569	pVmcsInfo->fWasInRealMode = false;
5570	}
5571	}
5572	#endif
5573	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5574	{
5575	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5576	if (pVmcsInfo->RealMode.fRealOnV86Active)
5577	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5578	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5579	AssertRCReturn(rc, rc);
5580	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5581	}
5582
5583	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5584	{
5585	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5586	if (pVmcsInfo->RealMode.fRealOnV86Active)
5587	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5588	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5589	AssertRCReturn(rc, rc);
5590	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5591	}
5592
5593	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5594	{
5595	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5596	if (pVmcsInfo->RealMode.fRealOnV86Active)
5597	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5598	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5599	AssertRCReturn(rc, rc);
5600	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5601	}
5602
5603	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5604	{
5605	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5606	if (pVmcsInfo->RealMode.fRealOnV86Active)
5607	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5608	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5609	AssertRCReturn(rc, rc);
5610	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5611	}
5612
5613	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5614	{
5615	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5616	if (pVmcsInfo->RealMode.fRealOnV86Active)
5617	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5618	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5619	AssertRCReturn(rc, rc);
5620	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5621	}
5622
5623	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5624	{
5625	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5626	if (pVmcsInfo->RealMode.fRealOnV86Active)
5627	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5628	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5629	AssertRCReturn(rc, rc);
5630	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5631	}
5632
5633	#ifdef VBOX_STRICT
5634	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5635	#endif
5636	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5637	pCtx->cs.Attr.u));
5638	}
5639
5640	/*
5641	* Guest TR.
5642	*/
5643	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5644	{
5645	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5646
5647	/*
5648	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5649	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5650	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5651	*/
5652	uint16_t u16Sel;
5653	uint32_t u32Limit;
5654	uint64_t u64Base;
5655	uint32_t u32AccessRights;
5656	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5657	{
5658	u16Sel = pCtx->tr.Sel;
5659	u32Limit = pCtx->tr.u32Limit;
5660	u64Base = pCtx->tr.u64Base;
5661	u32AccessRights = pCtx->tr.Attr.u;
5662	}
5663	else
5664	{
5665	Assert(!pVmxTransient->fIsNestedGuest);
5666	Assert(pVM->hm.s.vmx.pRealModeTSS);
5667	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5668
5669	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5670	RTGCPHYS GCPhys;
5671	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5672	AssertRCReturn(rc, rc);
5673
5674	X86DESCATTR DescAttr;
5675	DescAttr.u = 0;
5676	DescAttr.n.u1Present = 1;
5677	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5678
5679	u16Sel = 0;
5680	u32Limit = HM_VTX_TSS_SIZE;
5681	u64Base = GCPhys;
5682	u32AccessRights = DescAttr.u;
5683	}
5684
5685	/* Validate. */
5686	Assert(!(u16Sel & RT_BIT(2)));
5687	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5688	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5689	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5690	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5691	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5692	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5693	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5694	Assert( (u32Limit & 0xfff) == 0xfff
5695	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5696	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5697	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5698
5699	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5700	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5701	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5702	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5703	AssertRCReturn(rc, rc);
5704
5705	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5706	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5707	}
5708
5709	/*
5710	* Guest GDTR.
5711	*/
5712	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5713	{
5714	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5715
5716	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5717	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5718	AssertRCReturn(rc, rc);
5719
5720	/* Validate. */
5721	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5722
5723	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5724	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5725	}
5726
5727	/*
5728	* Guest LDTR.
5729	*/
5730	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5731	{
5732	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5733
5734	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5735	uint32_t u32Access;
5736	if ( !pVmxTransient->fIsNestedGuest
5737	&& !pCtx->ldtr.Attr.u)
5738	u32Access = X86DESCATTR_UNUSABLE;
5739	else
5740	u32Access = pCtx->ldtr.Attr.u;
5741
5742	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5743	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5744	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5745	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5746	AssertRCReturn(rc, rc);
5747
5748	/* Validate. */
5749	if (!(u32Access & X86DESCATTR_UNUSABLE))
5750	{
5751	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5752	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5753	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5754	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5755	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5756	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5757	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5758	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5759	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5760	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5761	}
5762
5763	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5764	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5765	}
5766
5767	/*
5768	* Guest IDTR.
5769	*/
5770	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5771	{
5772	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5773
5774	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5775	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5776	AssertRCReturn(rc, rc);
5777
5778	/* Validate. */
5779	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5780
5781	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5782	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5783	}
5784
5785	return VINF_SUCCESS;
5786	}
5787
5788
5789	/**
5790	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5791	* areas.
5792	*
5793	* These MSRs will automatically be loaded to the host CPU on every successful
5794	* VM-entry and stored from the host CPU on every successful VM-exit.
5795	*
5796	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5797	* actual host MSR values are not- updated here for performance reasons. See
5798	* hmR0VmxExportHostMsrs().
5799	*
5800	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5801	*
5802	* @returns VBox status code.
5803	* @param pVCpu The cross context virtual CPU structure.
5804	* @param pVmxTransient The VMX-transient structure.
5805	*
5806	* @remarks No-long-jump zone!!!
5807	*/
5808	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5809	{
5810	AssertPtr(pVCpu);
5811	AssertPtr(pVmxTransient);
5812
5813	PVM pVM = pVCpu->CTX_SUFF(pVM);
5814	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5815
5816	/*
5817	* MSRs that we use the auto-load/store MSR area in the VMCS.
5818	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5819	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5820	*
5821	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5822	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5823	* emulation, nothing to do here.
5824	*/
5825	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5826	{
5827	if ( !pVmxTransient->fIsNestedGuest
5828	&& pVM->hm.s.fAllow64BitGuests)
5829	{
5830	#if HC_ARCH_BITS == 32
5831	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5832	Assert(!pVmxTransient->fIsNestedGuest);
5833
5834	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5835	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5836	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5837	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5838	AssertRCReturn(rc, rc);
5839	#endif
5840	}
5841	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5842	}
5843
5844	/*
5845	* Guest Sysenter MSRs.
5846	*/
5847	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5848	{
5849	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5850
5851	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5852	{
5853	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5854	AssertRCReturn(rc, rc);
5855	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5856	}
5857
5858	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5859	{
5860	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5861	AssertRCReturn(rc, rc);
5862	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5863	}
5864
5865	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5866	{
5867	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5868	AssertRCReturn(rc, rc);
5869	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5870	}
5871	}
5872
5873	/*
5874	* Guest/host EFER MSR.
5875	*/
5876	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5877	{
5878	/* Whether we are using the VMCS to swap the EFER MSR must have been
5879	determined earlier while exporting VM-entry/VM-exit controls. */
5880	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5881	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5882
5883	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5884	{
5885	/*
5886	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5887	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5888	*/
5889	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5890	{
5891	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5892	AssertRCReturn(rc, rc);
5893	}
5894	else
5895	{
5896	/*
5897	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5898	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5899	*/
5900	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5901	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5902	AssertRCReturn(rc, rc);
5903	}
5904	}
5905	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5906	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5907
5908	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5909	}
5910
5911	/*
5912	* Other MSRs.
5913	* Speculation Control (R/W).
5914	*/
5915	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5916	{
5917	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5918	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5919	{
5920	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5921	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5922	AssertRCReturn(rc, rc);
5923	}
5924	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5925	}
5926
5927	return VINF_SUCCESS;
5928	}
5929
5930
5931	/**
5932	* Selects up the appropriate function to run guest code.
5933	*
5934	* @returns VBox status code.
5935	* @param pVCpu The cross context virtual CPU structure.
5936	* @param pVmxTransient The VMX-transient structure.
5937	*
5938	* @remarks No-long-jump zone!!!
5939	*/
5940	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5941	{
5942	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5943	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5944
5945	if (CPUMIsGuestInLongModeEx(pCtx))
5946	{
5947	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5948	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5949	#endif
5950	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5951	#if HC_ARCH_BITS == 32
5952	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5953	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5954	{
5955	#ifdef VBOX_STRICT
5956	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5957	{
5958	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5959	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5960	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5961	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5962	("fCtxChanged=%#RX64\n", fCtxChanged));
5963	}
5964	#endif
5965	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5966
5967	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5968	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5969	pVmcsInfo->fSwitchedTo64on32 = true;
5970	Log4Func(("Selected 64-bit switcher\n"));
5971	}
5972	#else
5973	/* 64-bit host. */
5974	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5975	#endif
5976	}
5977	else
5978	{
5979	/* Guest is not in long mode, use the 32-bit handler. */
5980	#if HC_ARCH_BITS == 32
5981	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5982	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5983	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5984	{
5985	# ifdef VBOX_STRICT
5986	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5987	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5988	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5989	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5990	("fCtxChanged=%#RX64\n", fCtxChanged));
5991	# endif
5992	}
5993	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5994	/*
5995	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5996	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5997	* switcher flag now because we know the guest is in a sane state where it's safe
5998	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5999	* the much faster 32-bit switcher again.
6000	*/
6001	if (!pVmcsInfo->fSwitchedTo64on32)
6002	{
6003	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6004	Log4Func(("Selected 32-bit switcher\n"));
6005	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6006	}
6007	else
6008	{
6009	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6010	if ( pVmcsInfo->RealMode.fRealOnV86Active
6011	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6012	{
6013	pVmcsInfo->fSwitchedTo64on32 = false;
6014	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6015	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6016	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6017	\| HM_CHANGED_HOST_CONTEXT);
6018	Log4Func(("Selected 32-bit switcher (safe)\n"));
6019	}
6020	}
6021	# else
6022	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6023	# endif
6024	#else
6025	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6026	#endif
6027	}
6028	Assert(pVmcsInfo->pfnStartVM);
6029	return VINF_SUCCESS;
6030	}
6031
6032
6033	/**
6034	* Wrapper for running the guest code in VT-x.
6035	*
6036	* @returns VBox status code, no informational status codes.
6037	* @param pVCpu The cross context virtual CPU structure.
6038	* @param pVmxTransient The VMX-transient structure.
6039	*
6040	* @remarks No-long-jump zone!!!
6041	*/
6042	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6043	{
6044	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6045	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6046	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6047
6048	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6049
6050	/*
6051	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6052	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6053	* callee-saved and thus the need for this XMM wrapper.
6054	*
6055	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6056	*/
6057	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6058	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6059	PVM pVM = pVCpu->CTX_SUFF(pVM);
6060	#ifdef VBOX_WITH_KERNEL_USING_XMM
6061	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6062	#else
6063	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6064	#endif
6065	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6066	return rc;
6067	}
6068
6069
6070	/**
6071	* Reports world-switch error and dumps some useful debug info.
6072	*
6073	* @param pVCpu The cross context virtual CPU structure.
6074	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6075	* @param pVmxTransient The VMX-transient structure (only
6076	* exitReason updated).
6077	*/
6078	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6079	{
6080	Assert(pVCpu);
6081	Assert(pVmxTransient);
6082	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6083
6084	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6085	switch (rcVMRun)
6086	{
6087	case VERR_VMX_INVALID_VMXON_PTR:
6088	AssertFailed();
6089	break;
6090	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6091	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6092	{
6093	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6094	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6095	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6096	AssertRC(rc);
6097
6098	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6099	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6100	Cannot do it here as we may have been long preempted. */
6101
6102	#ifdef VBOX_STRICT
6103	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6104	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6105	pVmxTransient->uExitReason));
6106	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6107	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6108	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6109	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6110	else
6111	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6112	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6113	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6114
6115	/* VMX control bits. */
6116	uint32_t u32Val;
6117	uint64_t u64Val;
6118	RTHCUINTREG uHCReg;
6119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6120	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6122	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6123	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6124	{
6125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6126	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6127	}
6128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6129	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6131	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6132	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6133	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6135	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6136	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6137	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6138	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6139	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6140	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6141	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6142	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6143	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6144	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6145	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6146	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6147	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6148	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6149	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6150	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6151	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6152	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6153	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6154	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6155	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6156	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6157	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6158	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6159	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6160	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6161	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6162	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6163	{
6164	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6165	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6166	}
6167
6168	/* Guest bits. */
6169	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6170	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6171	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6172	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6173	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6174	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6175	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6176	{
6177	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6178	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6179	}
6180
6181	/* Host bits. */
6182	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6183	Log4(("Host CR0 %#RHr\n", uHCReg));
6184	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6185	Log4(("Host CR3 %#RHr\n", uHCReg));
6186	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6187	Log4(("Host CR4 %#RHr\n", uHCReg));
6188
6189	RTGDTR HostGdtr;
6190	PCX86DESCHC pDesc;
6191	ASMGetGDTR(&HostGdtr);
6192	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6193	Log4(("Host CS %#08x\n", u32Val));
6194	if (u32Val < HostGdtr.cbGdt)
6195	{
6196	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6197	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6198	}
6199
6200	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6201	Log4(("Host DS %#08x\n", u32Val));
6202	if (u32Val < HostGdtr.cbGdt)
6203	{
6204	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6205	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6206	}
6207
6208	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6209	Log4(("Host ES %#08x\n", u32Val));
6210	if (u32Val < HostGdtr.cbGdt)
6211	{
6212	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6213	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6214	}
6215
6216	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6217	Log4(("Host FS %#08x\n", u32Val));
6218	if (u32Val < HostGdtr.cbGdt)
6219	{
6220	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6221	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6222	}
6223
6224	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6225	Log4(("Host GS %#08x\n", u32Val));
6226	if (u32Val < HostGdtr.cbGdt)
6227	{
6228	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6229	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6230	}
6231
6232	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6233	Log4(("Host SS %#08x\n", u32Val));
6234	if (u32Val < HostGdtr.cbGdt)
6235	{
6236	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6237	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6238	}
6239
6240	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6241	Log4(("Host TR %#08x\n", u32Val));
6242	if (u32Val < HostGdtr.cbGdt)
6243	{
6244	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6245	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6246	}
6247
6248	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6249	Log4(("Host TR Base %#RHv\n", uHCReg));
6250	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6251	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6252	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6253	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6254	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6255	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6256	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6257	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6258	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6259	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6260	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6261	Log4(("Host RSP %#RHv\n", uHCReg));
6262	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6263	Log4(("Host RIP %#RHv\n", uHCReg));
6264	# if HC_ARCH_BITS == 64
6265	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6266	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6267	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6268	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6269	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6270	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6271	# endif
6272	#endif /* VBOX_STRICT */
6273	break;
6274	}
6275
6276	default:
6277	/* Impossible */
6278	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6279	break;
6280	}
6281	}
6282
6283
6284	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6285	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6286	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6287	# endif
6288
6289	/**
6290	* Initialize the VMCS-Read cache.
6291	*
6292	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6293	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6294	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6295	* (those that have a 32-bit FULL & HIGH part).
6296	*
6297	* @param pVCpu The cross context virtual CPU structure.
6298	*/
6299	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6300	{
6301	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6302	do { \
6303	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6304	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6305	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6306	++cReadFields; \
6307	} while (0)
6308
6309	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6310	uint32_t cReadFields = 0;
6311
6312	/*
6313	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6314	* and serve to indicate exceptions to the rules.
6315	*/
6316
6317	/* Guest-natural selector base fields. */
6318	#if 0
6319	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6322	#endif
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6334	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6335	#if 0
6336	/* Unused natural width guest-state fields. */
6337	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6338	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6339	#endif
6340	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6341	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6342
6343	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6344	these 64-bit fields (using "FULL" and "HIGH" fields). */
6345	#if 0
6346	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6347	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6348	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6349	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6350	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6351	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6352	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6353	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6354	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6355	#endif
6356
6357	/* Natural width guest-state fields. */
6358	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6359	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6360
6361	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6362	{
6363	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6364	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6365	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6366	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6367	}
6368	else
6369	{
6370	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6371	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6372	}
6373
6374	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6375	}
6376
6377
6378	/**
6379	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6380	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6381	* darwin, running 64-bit guests).
6382	*
6383	* @returns VBox status code.
6384	* @param pVCpu The cross context virtual CPU structure.
6385	* @param idxField The VMCS field encoding.
6386	* @param u64Val 16, 32 or 64-bit value.
6387	*/
6388	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6389	{
6390	int rc;
6391	switch (idxField)
6392	{
6393	/*
6394	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6395	*/
6396	/* 64-bit Control fields. */
6397	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6398	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6399	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6400	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6401	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6402	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6403	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6404	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6405	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6406	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6407	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6408	case VMX_VMCS64_CTRL_EPTP_FULL:
6409	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6410	/* 64-bit Guest-state fields. */
6411	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6412	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6413	case VMX_VMCS64_GUEST_PAT_FULL:
6414	case VMX_VMCS64_GUEST_EFER_FULL:
6415	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6416	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6417	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6418	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6419	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6420	/* 64-bit Host-state fields. */
6421	case VMX_VMCS64_HOST_PAT_FULL:
6422	case VMX_VMCS64_HOST_EFER_FULL:
6423	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6424	{
6425	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6426	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6427	break;
6428	}
6429
6430	/*
6431	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6432	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6433	*/
6434	/* Natural-width Guest-state fields. */
6435	case VMX_VMCS_GUEST_CR3:
6436	case VMX_VMCS_GUEST_ES_BASE:
6437	case VMX_VMCS_GUEST_CS_BASE:
6438	case VMX_VMCS_GUEST_SS_BASE:
6439	case VMX_VMCS_GUEST_DS_BASE:
6440	case VMX_VMCS_GUEST_FS_BASE:
6441	case VMX_VMCS_GUEST_GS_BASE:
6442	case VMX_VMCS_GUEST_LDTR_BASE:
6443	case VMX_VMCS_GUEST_TR_BASE:
6444	case VMX_VMCS_GUEST_GDTR_BASE:
6445	case VMX_VMCS_GUEST_IDTR_BASE:
6446	case VMX_VMCS_GUEST_RSP:
6447	case VMX_VMCS_GUEST_RIP:
6448	case VMX_VMCS_GUEST_SYSENTER_ESP:
6449	case VMX_VMCS_GUEST_SYSENTER_EIP:
6450	{
6451	if (!(RT_HI_U32(u64Val)))
6452	{
6453	/* If this field is 64-bit, VT-x will zero out the top bits. */
6454	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6455	}
6456	else
6457	{
6458	/* Assert that only the 32->64 switcher case should ever come here. */
6459	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6460	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6461	}
6462	break;
6463	}
6464
6465	default:
6466	{
6467	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6468	pVCpu->hm.s.u32HMError = idxField;
6469	rc = VERR_INVALID_PARAMETER;
6470	break;
6471	}
6472	}
6473	AssertRCReturn(rc, rc);
6474	return rc;
6475	}
6476
6477
6478	/**
6479	* Queue up a VMWRITE by using the VMCS write cache.
6480	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6481	*
6482	* @param pVCpu The cross context virtual CPU structure.
6483	* @param idxField The VMCS field encoding.
6484	* @param u64Val 16, 32 or 64-bit value.
6485	*/
6486	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6487	{
6488	AssertPtr(pVCpu);
6489	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6490
6491	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6492	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6493
6494	/* Make sure there are no duplicates. */
6495	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6496	{
6497	if (pCache->Write.aField[i] == idxField)
6498	{
6499	pCache->Write.aFieldVal[i] = u64Val;
6500	return VINF_SUCCESS;
6501	}
6502	}
6503
6504	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6505	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6506	pCache->Write.cValidEntries++;
6507	return VINF_SUCCESS;
6508	}
6509	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6510
6511
6512	/**
6513	* Sets up the usage of TSC-offsetting and updates the VMCS.
6514	*
6515	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6516	* VMX-preemption timer.
6517	*
6518	* @returns VBox status code.
6519	* @param pVCpu The cross context virtual CPU structure.
6520	* @param pVmxTransient The VMX-transient structure.
6521	*
6522	* @remarks No-long-jump zone!!!
6523	*/
6524	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6525	{
6526	bool fOffsettedTsc;
6527	bool fParavirtTsc;
6528	uint64_t uTscOffset;
6529	PVM pVM = pVCpu->CTX_SUFF(pVM);
6530	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6531
6532	if (pVM->hm.s.vmx.fUsePreemptTimer)
6533	{
6534	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6535
6536	/* Make sure the returned values have sane upper and lower boundaries. */
6537	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6538	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6539	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6540	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6541
6542	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6543	* preemption timers here. We probably need to clamp the preemption timer,
6544	* after converting the timer value to the host. */
6545	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6546	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6547	AssertRC(rc);
6548	}
6549	else
6550	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6551
6552	if (fParavirtTsc)
6553	{
6554	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6555	information before every VM-entry, hence disable it for performance sake. */
6556	#if 0
6557	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6558	AssertRC(rc);
6559	#endif
6560	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6561	}
6562
6563	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6564	if ( fOffsettedTsc
6565	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6566	{
6567	if (pVmxTransient->fIsNestedGuest)
6568	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6569	if (pVmcsInfo->u64TscOffset != uTscOffset)
6570	{
6571	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6572	AssertRC(rc);
6573	pVmcsInfo->u64TscOffset = uTscOffset;
6574	}
6575
6576	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6577	{
6578	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6579	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6580	AssertRC(rc);
6581	pVmcsInfo->u32ProcCtls = uProcCtls;
6582	}
6583	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6584	}
6585	else
6586	{
6587	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6588	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6589	{
6590	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6591	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6592	AssertRC(rc);
6593	pVmcsInfo->u32ProcCtls = uProcCtls;
6594	}
6595	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6596	}
6597	}
6598
6599
6600	/**
6601	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6602	* VM-exit interruption info type.
6603	*
6604	* @returns The IEM exception flags.
6605	* @param uVector The event vector.
6606	* @param uVmxEventType The VMX event type.
6607	*
6608	* @remarks This function currently only constructs flags required for
6609	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6610	* and CR2 aspects of an exception are not included).
6611	*/
6612	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6613	{
6614	uint32_t fIemXcptFlags;
6615	switch (uVmxEventType)
6616	{
6617	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6618	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6619	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6620	break;
6621
6622	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6623	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6624	break;
6625
6626	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6627	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6628	break;
6629
6630	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6631	{
6632	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6633	if (uVector == X86_XCPT_BP)
6634	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6635	else if (uVector == X86_XCPT_OF)
6636	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6637	else
6638	{
6639	fIemXcptFlags = 0;
6640	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6641	}
6642	break;
6643	}
6644
6645	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6646	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6647	break;
6648
6649	default:
6650	fIemXcptFlags = 0;
6651	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6652	break;
6653	}
6654	return fIemXcptFlags;
6655	}
6656
6657
6658	/**
6659	* Sets an event as a pending event to be injected into the guest.
6660	*
6661	* @param pVCpu The cross context virtual CPU structure.
6662	* @param u32IntInfo The VM-entry interruption-information field.
6663	* @param cbInstr The VM-entry instruction length in bytes (for software
6664	* interrupts, exceptions and privileged software
6665	* exceptions).
6666	* @param u32ErrCode The VM-entry exception error code.
6667	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6668	* page-fault.
6669	*/
6670	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6671	RTGCUINTPTR GCPtrFaultAddress)
6672	{
6673	Assert(!pVCpu->hm.s.Event.fPending);
6674	pVCpu->hm.s.Event.fPending = true;
6675	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6676	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6677	pVCpu->hm.s.Event.cbInstr = cbInstr;
6678	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6679	}
6680
6681
6682	/**
6683	* Sets an external interrupt as pending-for-injection into the VM.
6684	*
6685	* @param pVCpu The cross context virtual CPU structure.
6686	* @param u8Interrupt The external interrupt vector.
6687	*/
6688	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6689	{
6690	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6691	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6692	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6693	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6694	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6695	}
6696
6697
6698	/**
6699	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6700	*
6701	* @param pVCpu The cross context virtual CPU structure.
6702	*/
6703	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6704	{
6705	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6706	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6707	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6708	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6709	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6710	}
6711
6712
6713	/**
6714	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6715	*
6716	* @param pVCpu The cross context virtual CPU structure.
6717	*/
6718	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6719	{
6720	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6721	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6722	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6723	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6724	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6725	}
6726
6727
6728	/**
6729	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6730	*
6731	* @param pVCpu The cross context virtual CPU structure.
6732	*/
6733	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6734	{
6735	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6736	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6737	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6738	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6739	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6740	}
6741
6742
6743	/**
6744	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6745	*
6746	* @param pVCpu The cross context virtual CPU structure.
6747	*/
6748	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6749	{
6750	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6751	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6752	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6753	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6754	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6755	}
6756
6757
6758	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6759	/**
6760	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6761	*
6762	* @param pVCpu The cross context virtual CPU structure.
6763	* @param u32ErrCode The error code for the general-protection exception.
6764	*/
6765	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6766	{
6767	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6768	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6769	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6770	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6771	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6772	}
6773
6774
6775	/**
6776	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6777	*
6778	* @param pVCpu The cross context virtual CPU structure.
6779	* @param u32ErrCode The error code for the stack exception.
6780	*/
6781	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6782	{
6783	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6784	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6785	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6786	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6787	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6788	}
6789
6790
6791	/**
6792	* Decodes the memory operand of an instruction that caused a VM-exit.
6793	*
6794	* The VM-exit qualification field provides the displacement field for memory
6795	* operand instructions, if any.
6796	*
6797	* @returns Strict VBox status code (i.e. informational status codes too).
6798	* @retval VINF_SUCCESS if the operand was successfully decoded.
6799	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6800	* operand.
6801	* @param pVCpu The cross context virtual CPU structure.
6802	* @param uExitInstrInfo The VM-exit instruction information field.
6803	* @param enmMemAccess The memory operand's access type (read or write).
6804	* @param GCPtrDisp The instruction displacement field, if any. For
6805	* RIP-relative addressing pass RIP + displacement here.
6806	* @param pGCPtrMem Where to store the effective destination memory address.
6807	*
6808	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6809	* virtual-8086 mode hence skips those checks while verifying if the
6810	* segment is valid.
6811	*/
6812	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6813	PRTGCPTR pGCPtrMem)
6814	{
6815	Assert(pGCPtrMem);
6816	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6817	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6818	\| CPUMCTX_EXTRN_CR0);
6819
6820	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6821	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6822	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6823
6824	VMXEXITINSTRINFO ExitInstrInfo;
6825	ExitInstrInfo.u = uExitInstrInfo;
6826	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6827	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6828	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6829	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6830	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6831	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6832	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6833	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6834	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6835
6836	/*
6837	* Validate instruction information.
6838	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6839	*/
6840	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6841	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6842	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6843	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6844	AssertLogRelMsgReturn(fIsMemOperand,
6845	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6846
6847	/*
6848	* Compute the complete effective address.
6849	*
6850	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6851	* See AMD spec. 4.5.2 "Segment Registers".
6852	*/
6853	RTGCPTR GCPtrMem = GCPtrDisp;
6854	if (fBaseRegValid)
6855	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6856	if (fIdxRegValid)
6857	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6858
6859	RTGCPTR const GCPtrOff = GCPtrMem;
6860	if ( !fIsLongMode
6861	\|\| iSegReg >= X86_SREG_FS)
6862	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6863	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6864
6865	/*
6866	* Validate effective address.
6867	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6868	*/
6869	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6870	Assert(cbAccess > 0);
6871	if (fIsLongMode)
6872	{
6873	if (X86_IS_CANONICAL(GCPtrMem))
6874	{
6875	*pGCPtrMem = GCPtrMem;
6876	return VINF_SUCCESS;
6877	}
6878
6879	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6880	* "Data Limit Checks in 64-bit Mode". */
6881	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6882	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6883	return VINF_HM_PENDING_XCPT;
6884	}
6885
6886	/*
6887	* This is a watered down version of iemMemApplySegment().
6888	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6889	* and segment CPL/DPL checks are skipped.
6890	*/
6891	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6892	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6893	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6894
6895	/* Check if the segment is present and usable. */
6896	if ( pSel->Attr.n.u1Present
6897	&& !pSel->Attr.n.u1Unusable)
6898	{
6899	Assert(pSel->Attr.n.u1DescType);
6900	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6901	{
6902	/* Check permissions for the data segment. */
6903	if ( enmMemAccess == VMXMEMACCESS_WRITE
6904	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6905	{
6906	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6907	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6908	return VINF_HM_PENDING_XCPT;
6909	}
6910
6911	/* Check limits if it's a normal data segment. */
6912	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6913	{
6914	if ( GCPtrFirst32 > pSel->u32Limit
6915	\|\| GCPtrLast32 > pSel->u32Limit)
6916	{
6917	Log4Func(("Data segment limit exceeded. "
6918	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6919	GCPtrLast32, pSel->u32Limit));
6920	if (iSegReg == X86_SREG_SS)
6921	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6922	else
6923	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6924	return VINF_HM_PENDING_XCPT;
6925	}
6926	}
6927	else
6928	{
6929	/* Check limits if it's an expand-down data segment.
6930	Note! The upper boundary is defined by the B bit, not the G bit! */
6931	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6932	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6933	{
6934	Log4Func(("Expand-down data segment limit exceeded. "
6935	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6936	GCPtrLast32, pSel->u32Limit));
6937	if (iSegReg == X86_SREG_SS)
6938	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6939	else
6940	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6941	return VINF_HM_PENDING_XCPT;
6942	}
6943	}
6944	}
6945	else
6946	{
6947	/* Check permissions for the code segment. */
6948	if ( enmMemAccess == VMXMEMACCESS_WRITE
6949	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6950	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6951	{
6952	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6953	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6954	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6955	return VINF_HM_PENDING_XCPT;
6956	}
6957
6958	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6959	if ( GCPtrFirst32 > pSel->u32Limit
6960	\|\| GCPtrLast32 > pSel->u32Limit)
6961	{
6962	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6963	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6964	if (iSegReg == X86_SREG_SS)
6965	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6966	else
6967	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6968	return VINF_HM_PENDING_XCPT;
6969	}
6970	}
6971	}
6972	else
6973	{
6974	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6975	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6976	return VINF_HM_PENDING_XCPT;
6977	}
6978
6979	*pGCPtrMem = GCPtrMem;
6980	return VINF_SUCCESS;
6981	}
6982
6983
6984	/**
6985	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6986	* guest attempting to execute a VMX instruction.
6987	*
6988	* @returns Strict VBox status code (i.e. informational status codes too).
6989	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6990	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6991	*
6992	* @param pVCpu The cross context virtual CPU structure.
6993	* @param uExitReason The VM-exit reason.
6994	*
6995	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6996	* @remarks No-long-jump zone!!!
6997	*/
6998	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6999	{
7000	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
7001	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7002
7003	/*
7004	* The physical CPU would have already checked the CPU mode/code segment.
7005	* We shall just assert here for paranoia.
7006	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7007	*/
7008	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7009	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7010	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7011
7012	if (uExitReason == VMX_EXIT_VMXON)
7013	{
7014	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7015
7016	/*
7017	* We check CR4.VMXE because it is required to be always set while in VMX operation
7018	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
7019	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7020	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7021	*/
7022	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7023	{
7024	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7025	hmR0VmxSetPendingXcptUD(pVCpu);
7026	return VINF_HM_PENDING_XCPT;
7027	}
7028	}
7029	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7030	{
7031	/*
7032	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7033	* (other than VMXON), we need to raise a #UD.
7034	*/
7035	Log4Func(("Not in VMX root mode -> #UD\n"));
7036	hmR0VmxSetPendingXcptUD(pVCpu);
7037	return VINF_HM_PENDING_XCPT;
7038	}
7039
7040	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7041	return VINF_SUCCESS;
7042	}
7043	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7044
7045
7046	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7047	{
7048	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7049
7050	/*
7051	* If VT-x marks the segment as unusable, most other bits remain undefined:
7052	* - For CS the L, D and G bits have meaning.
7053	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7054	* - For the remaining data segments no bits are defined.
7055	*
7056	* The present bit and the unusable bit has been observed to be set at the
7057	* same time (the selector was supposed to be invalid as we started executing
7058	* a V8086 interrupt in ring-0).
7059	*
7060	* What should be important for the rest of the VBox code, is that the P bit is
7061	* cleared. Some of the other VBox code recognizes the unusable bit, but
7062	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7063	* safe side here, we'll strip off P and other bits we don't care about. If
7064	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7065	*
7066	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7067	*/
7068	#ifdef VBOX_STRICT
7069	uint32_t const uAttr = pSelReg->Attr.u;
7070	#endif
7071
7072	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7073	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7074	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7075
7076	#ifdef VBOX_STRICT
7077	VMMRZCallRing3Disable(pVCpu);
7078	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7079	# ifdef DEBUG_bird
7080	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7081	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7082	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7083	# endif
7084	VMMRZCallRing3Enable(pVCpu);
7085	NOREF(uAttr);
7086	#endif
7087	RT_NOREF2(pVCpu, idxSel);
7088	}
7089
7090
7091	/**
7092	* Imports a guest segment register from the current VMCS into the guest-CPU
7093	* context.
7094	*
7095	* @returns VBox status code.
7096	* @param pVCpu The cross context virtual CPU structure.
7097	* @param iSegReg The segment register number (X86_SREG_XXX).
7098	*
7099	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7100	* do not log!
7101	*/
7102	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7103	{
7104	Assert(iSegReg < X86_SREG_COUNT);
7105
7106	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7107	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7108	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7109	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7110	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7111	#else
7112	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7113	#endif
7114	uint64_t u64Base;
7115	uint32_t u32Sel, u32Limit, u32Attr;
7116	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7117	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7118	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7119	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7120	if (RT_SUCCESS(rc))
7121	{
7122	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7123	pSelReg->Sel = u32Sel;
7124	pSelReg->ValidSel = u32Sel;
7125	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7126	pSelReg->u32Limit = u32Limit;
7127	pSelReg->u64Base = u64Base;
7128	pSelReg->Attr.u = u32Attr;
7129	if (u32Attr & X86DESCATTR_UNUSABLE)
7130	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7131	}
7132	return rc;
7133	}
7134
7135
7136	/**
7137	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7138	*
7139	* @returns VBox status code.
7140	* @param pVCpu The cross context virtual CPU structure.
7141	*
7142	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7143	* do not log!
7144	*/
7145	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7146	{
7147	uint64_t u64Base;
7148	uint32_t u32Sel, u32Limit, u32Attr;
7149	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7150	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7151	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7152	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7153	if (RT_SUCCESS(rc))
7154	{
7155	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7156	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7157	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7158	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7159	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7160	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7161	if (u32Attr & X86DESCATTR_UNUSABLE)
7162	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7163	}
7164	return rc;
7165	}
7166
7167
7168	/**
7169	* Imports the guest TR from the current VMCS into the guest-CPU context.
7170	*
7171	* @returns VBox status code.
7172	* @param pVCpu The cross context virtual CPU structure.
7173	*
7174	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7175	* do not log!
7176	*/
7177	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7178	{
7179	uint32_t u32Sel, u32Limit, u32Attr;
7180	uint64_t u64Base;
7181	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7182	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7183	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7184	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7185	AssertRCReturn(rc, rc);
7186
7187	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7188	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7189	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7190	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7191	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7192	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7193	/* TR is the only selector that can never be unusable. */
7194	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7195	return VINF_SUCCESS;
7196	}
7197
7198
7199	/**
7200	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7201	*
7202	* @returns VBox status code.
7203	* @param pVCpu The cross context virtual CPU structure.
7204	*
7205	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7206	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7207	* instead!!!
7208	*/
7209	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7210	{
7211	uint64_t u64Val;
7212	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7213	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7214	{
7215	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7216	if (RT_SUCCESS(rc))
7217	{
7218	pCtx->rip = u64Val;
7219	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7220	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7221	}
7222	return rc;
7223	}
7224	return VINF_SUCCESS;
7225	}
7226
7227
7228	/**
7229	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7230	*
7231	* @returns VBox status code.
7232	* @param pVCpu The cross context virtual CPU structure.
7233	* @param pVmcsInfo The VMCS info. object.
7234	*
7235	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7236	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7237	* instead!!!
7238	*/
7239	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7240	{
7241	uint32_t u32Val;
7242	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7243	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7244	{
7245	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7246	if (RT_SUCCESS(rc))
7247	{
7248	pCtx->eflags.u32 = u32Val;
7249
7250	/* Restore eflags for real-on-v86-mode hack. */
7251	if (pVmcsInfo->RealMode.fRealOnV86Active)
7252	{
7253	pCtx->eflags.Bits.u1VM = 0;
7254	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7255	}
7256	}
7257	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7258	return rc;
7259	}
7260	return VINF_SUCCESS;
7261	}
7262
7263
7264	/**
7265	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7266	* context.
7267	*
7268	* @returns VBox status code.
7269	* @param pVCpu The cross context virtual CPU structure.
7270	* @param pVmcsInfo The VMCS info. object.
7271	*
7272	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7273	* do not log!
7274	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7275	* instead!!!
7276	*/
7277	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7278	{
7279	uint32_t u32Val;
7280	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7281	if (RT_SUCCESS(rc))
7282	{
7283	if (!u32Val)
7284	{
7285	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7286	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7287
7288	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7289	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7290	}
7291	else
7292	{
7293	/*
7294	* We must import RIP here to set our EM interrupt-inhibited state.
7295	* We also import RFLAGS as our code that evaluates pending interrupts
7296	* before VM-entry requires it.
7297	*/
7298	rc = hmR0VmxImportGuestRip(pVCpu);
7299	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7300	if (RT_SUCCESS(rc))
7301	{
7302	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7303	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7304	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7305	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7306
7307	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7308	{
7309	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7310	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7311	}
7312	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7313	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7314	}
7315	}
7316	}
7317	return rc;
7318	}
7319
7320
7321	/**
7322	* Worker for VMXR0ImportStateOnDemand.
7323	*
7324	* @returns VBox status code.
7325	* @param pVCpu The cross context virtual CPU structure.
7326	* @param pVmcsInfo The VMCS info. object.
7327	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7328	*/
7329	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7330	{
7331	#define VMXLOCAL_BREAK_RC(a_rc) \
7332	if (RT_SUCCESS(a_rc)) \
7333	{ } \
7334	else \
7335	break
7336
7337	int rc = VINF_SUCCESS;
7338	PVM pVM = pVCpu->CTX_SUFF(pVM);
7339	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7340	uint64_t u64Val;
7341	uint32_t u32Val;
7342
7343	/*
7344	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7345	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7346	* neither are other host platforms.
7347	*
7348	* Committing this temporarily as it prevents BSOD.
7349	*
7350	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7351	*/
7352	#ifdef RT_OS_WINDOWS
7353	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7354	return VERR_HM_IPE_1;
7355	#endif
7356
7357	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7358
7359	/*
7360	* We disable interrupts to make the updating of the state and in particular
7361	* the fExtrn modification atomic wrt to preemption hooks.
7362	*/
7363	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7364
7365	fWhat &= pCtx->fExtrn;
7366	if (fWhat)
7367	{
7368	do
7369	{
7370	if (fWhat & CPUMCTX_EXTRN_RIP)
7371	{
7372	rc = hmR0VmxImportGuestRip(pVCpu);
7373	VMXLOCAL_BREAK_RC(rc);
7374	}
7375
7376	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7377	{
7378	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7379	VMXLOCAL_BREAK_RC(rc);
7380	}
7381
7382	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7383	{
7384	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7385	VMXLOCAL_BREAK_RC(rc);
7386	}
7387
7388	if (fWhat & CPUMCTX_EXTRN_RSP)
7389	{
7390	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7391	VMXLOCAL_BREAK_RC(rc);
7392	pCtx->rsp = u64Val;
7393	}
7394
7395	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7396	{
7397	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7398	if (fWhat & CPUMCTX_EXTRN_CS)
7399	{
7400	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7401	rc \|= hmR0VmxImportGuestRip(pVCpu);
7402	if (fRealOnV86Active)
7403	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7404	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7405	}
7406	if (fWhat & CPUMCTX_EXTRN_SS)
7407	{
7408	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7409	if (fRealOnV86Active)
7410	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7411	}
7412	if (fWhat & CPUMCTX_EXTRN_DS)
7413	{
7414	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7415	if (fRealOnV86Active)
7416	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7417	}
7418	if (fWhat & CPUMCTX_EXTRN_ES)
7419	{
7420	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7421	if (fRealOnV86Active)
7422	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7423	}
7424	if (fWhat & CPUMCTX_EXTRN_FS)
7425	{
7426	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7427	if (fRealOnV86Active)
7428	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7429	}
7430	if (fWhat & CPUMCTX_EXTRN_GS)
7431	{
7432	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7433	if (fRealOnV86Active)
7434	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7435	}
7436	VMXLOCAL_BREAK_RC(rc);
7437	}
7438
7439	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7440	{
7441	if (fWhat & CPUMCTX_EXTRN_LDTR)
7442	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7443
7444	if (fWhat & CPUMCTX_EXTRN_GDTR)
7445	{
7446	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7447	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7448	pCtx->gdtr.pGdt = u64Val;
7449	pCtx->gdtr.cbGdt = u32Val;
7450	}
7451
7452	/* Guest IDTR. */
7453	if (fWhat & CPUMCTX_EXTRN_IDTR)
7454	{
7455	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7456	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7457	pCtx->idtr.pIdt = u64Val;
7458	pCtx->idtr.cbIdt = u32Val;
7459	}
7460
7461	/* Guest TR. */
7462	if (fWhat & CPUMCTX_EXTRN_TR)
7463	{
7464	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7465	don't need to import that one. */
7466	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7467	rc \|= hmR0VmxImportGuestTr(pVCpu);
7468	}
7469	VMXLOCAL_BREAK_RC(rc);
7470	}
7471
7472	if (fWhat & CPUMCTX_EXTRN_DR7)
7473	{
7474	if (!pVCpu->hm.s.fUsingHyperDR7)
7475	{
7476	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7477	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7478	VMXLOCAL_BREAK_RC(rc);
7479	pCtx->dr[7] = u32Val;
7480	}
7481	}
7482
7483	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7484	{
7485	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7486	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7487	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7488	pCtx->SysEnter.cs = u32Val;
7489	VMXLOCAL_BREAK_RC(rc);
7490	}
7491
7492	#if HC_ARCH_BITS == 64
7493	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7494	{
7495	if ( pVM->hm.s.fAllow64BitGuests
7496	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7497	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7498	}
7499
7500	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7501	{
7502	if ( pVM->hm.s.fAllow64BitGuests
7503	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7504	{
7505	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7506	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7507	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7508	}
7509	}
7510	#endif
7511
7512	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7513	#if HC_ARCH_BITS == 32
7514	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7515	#endif
7516	)
7517	{
7518	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7519	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7520	Assert(pMsrs);
7521	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7522	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7523	for (uint32_t i = 0; i < cMsrs; i++)
7524	{
7525	uint32_t const idMsr = pMsrs[i].u32Msr;
7526	switch (idMsr)
7527	{
7528	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7529	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7530	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7531	#if HC_ARCH_BITS == 32
7532	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7533	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7534	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7535	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7536	#endif
7537	default:
7538	{
7539	pCtx->fExtrn = 0;
7540	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7541	ASMSetFlags(fEFlags);
7542	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7543	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7544	}
7545	}
7546	}
7547	}
7548
7549	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7550	{
7551	uint64_t u64Shadow;
7552	if (fWhat & CPUMCTX_EXTRN_CR0)
7553	{
7554	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7555	* remove when we drop 32-bit host w/ 64-bit host support, see
7556	* @bugref{9180#c39}. */
7557	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7558	#if HC_ARCH_BITS == 32
7559	uint32_t u32Shadow;
7560	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7561	u64Shadow = u32Shadow;
7562	#else
7563	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7564	#endif
7565	VMXLOCAL_BREAK_RC(rc);
7566	u64Val = u32Val;
7567	#if 1
7568	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7569	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7570	#else
7571	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7572	{
7573	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7574	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7575	}
7576	else
7577	{
7578	/** @todo NSTVMX: We need to do some unfudging here because we altered the
7579	* guest/host mask before running the nested-guest. */
7580	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7581	Assert(pVmcsNstGst);
7582
7583	uint64_t const uGstCr0Mask = pVmcsNstGst->u64Cr0Mask.u;
7584	uint64_t const uHstCr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
7585	}
7586	#endif
7587	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7588	CPUMSetGuestCR0(pVCpu, u64Val);
7589	VMMRZCallRing3Enable(pVCpu);
7590	}
7591
7592	if (fWhat & CPUMCTX_EXTRN_CR4)
7593	{
7594	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7595	* remove when we drop 32-bit host w/ 64-bit host support, see
7596	* @bugref{9180#c39}. */
7597	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7598	#if HC_ARCH_BITS == 32
7599	uint32_t u32Shadow;
7600	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7601	u64Shadow = u32Shadow;
7602	#else
7603	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7604	#endif
7605	VMXLOCAL_BREAK_RC(rc);
7606	u64Val = u32Val;
7607	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7608	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7609	pCtx->cr4 = u64Val;
7610	}
7611
7612	if (fWhat & CPUMCTX_EXTRN_CR3)
7613	{
7614	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7615	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7616	\|\| ( pVM->hm.s.fNestedPaging
7617	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7618	{
7619	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7620	VMXLOCAL_BREAK_RC(rc);
7621	if (pCtx->cr3 != u64Val)
7622	{
7623	pCtx->cr3 = u64Val;
7624	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7625	}
7626
7627	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7628	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7629	if (CPUMIsGuestInPAEModeEx(pCtx))
7630	{
7631	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7632	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7633	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7634	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7635	VMXLOCAL_BREAK_RC(rc);
7636	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7637	}
7638	}
7639	}
7640
7641	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7642	# if 0
7643	/** @todo NSTVMX: We handle each of these fields individually by passing it to IEM
7644	* VM-exit handlers. We might handle it differently when using the fast path. */
7645	/*
7646	* The hardware virtualization state currently consists of VMCS fields that may be
7647	* modified by execution of the nested-guest (that are not part of the general
7648	* guest state) and is visible to guest software. Hence, it is technically part of
7649	* the guest-CPU state when executing a nested-guest.
7650	*/
7651	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7652	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7653	{
7654	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7655	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7656	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7657	VMXLOCAL_BREAK_RC(rc);
7658
7659	/*
7660	* VM-entry can fail due to invalid-guest state, machine-check events and
7661	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7662	* all other VMCS fields are left unmodified on VM-entry failure.
7663	*
7664	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7665	*/
7666	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7667	if (!fEntryFailed)
7668	{
7669	/*
7670	* Some notes on VMCS fields that may need importing when the fast path
7671	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7672	*
7673	* Requires fixing up when using hardware-assisted VMX:
7674	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7675	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7676	* - IDT-vectoring info: Think about this.
7677	* - IDT-vectoring error code: Think about this.
7678	*
7679	* Emulated:
7680	* - Guest-interruptiblity state: Derived from FFs and RIP.
7681	* - Guest pending debug exceptions: Derived from DR6.
7682	* - Guest activity state: Emulated from EM state.
7683	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7684	* - Entry-interrupt info: Emulated, cleared to 0.
7685	*/
7686	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7687	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7688	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7689	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7690	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7691	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7692	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7693	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7694	/** @todo NSTVMX: Save and adjust preemption timer value. */
7695	}
7696
7697	VMXLOCAL_BREAK_RC(rc);
7698	}
7699	# endif
7700	#endif
7701	}
7702	} while (0);
7703
7704	if (RT_SUCCESS(rc))
7705	{
7706	/* Update fExtrn. */
7707	pCtx->fExtrn &= ~fWhat;
7708
7709	/* If everything has been imported, clear the HM keeper bit. */
7710	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7711	{
7712	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7713	Assert(!pCtx->fExtrn);
7714	}
7715	}
7716	}
7717	else
7718	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7719
7720	ASMSetFlags(fEFlags);
7721
7722	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7723
7724	if (RT_SUCCESS(rc))
7725	{ /* likely */ }
7726	else
7727	return rc;
7728
7729	/*
7730	* Honor any pending CR3 updates.
7731	*
7732	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7733	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7734	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7735	*
7736	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7737	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7738	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7739	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7740	*
7741	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7742	*/
7743	if (VMMRZCallRing3IsEnabled(pVCpu))
7744	{
7745	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7746	{
7747	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7748	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7749	}
7750
7751	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7752	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7753
7754	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7755	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7756	}
7757
7758	return VINF_SUCCESS;
7759	#undef VMXLOCAL_BREAK_RC
7760	}
7761
7762
7763	/**
7764	* Saves the guest state from the VMCS into the guest-CPU context.
7765	*
7766	* @returns VBox status code.
7767	* @param pVCpu The cross context virtual CPU structure.
7768	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7769	*/
7770	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7771	{
7772	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7773	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7774	}
7775
7776
7777	/**
7778	* Check per-VM and per-VCPU force flag actions that require us to go back to
7779	* ring-3 for one reason or another.
7780	*
7781	* @returns Strict VBox status code (i.e. informational status codes too)
7782	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7783	* ring-3.
7784	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7785	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7786	* interrupts)
7787	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7788	* all EMTs to be in ring-3.
7789	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7790	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7791	* to the EM loop.
7792	*
7793	* @param pVCpu The cross context virtual CPU structure.
7794	* @param fStepping Whether we are single-stepping the guest using the
7795	* hypervisor debugger.
7796	*/
7797	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7798	{
7799	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7800
7801	/*
7802	* Update pending interrupts into the APIC's IRR.
7803	*/
7804	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7805	APICUpdatePendingInterrupts(pVCpu);
7806
7807	/*
7808	* Anything pending? Should be more likely than not if we're doing a good job.
7809	*/
7810	PVM pVM = pVCpu->CTX_SUFF(pVM);
7811	if ( !fStepping
7812	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7813	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7814	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7815	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7816	return VINF_SUCCESS;
7817
7818	/* Pending PGM C3 sync. */
7819	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7820	{
7821	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7822	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7823	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7824	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7825	if (rcStrict2 != VINF_SUCCESS)
7826	{
7827	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7828	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7829	return rcStrict2;
7830	}
7831	}
7832
7833	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7834	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7835	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7836	{
7837	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7838	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7839	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7840	return rc2;
7841	}
7842
7843	/* Pending VM request packets, such as hardware interrupts. */
7844	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7845	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7846	{
7847	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7848	return VINF_EM_PENDING_REQUEST;
7849	}
7850
7851	/* Pending PGM pool flushes. */
7852	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7853	{
7854	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7855	return VINF_PGM_POOL_FLUSH_PENDING;
7856	}
7857
7858	/* Pending DMA requests. */
7859	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7860	{
7861	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7862	return VINF_EM_RAW_TO_R3;
7863	}
7864
7865	return VINF_SUCCESS;
7866	}
7867
7868
7869	/**
7870	* Converts any TRPM trap into a pending HM event. This is typically used when
7871	* entering from ring-3 (not longjmp returns).
7872	*
7873	* @param pVCpu The cross context virtual CPU structure.
7874	*/
7875	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7876	{
7877	Assert(TRPMHasTrap(pVCpu));
7878	Assert(!pVCpu->hm.s.Event.fPending);
7879
7880	uint8_t uVector;
7881	TRPMEVENT enmTrpmEvent;
7882	RTGCUINT uErrCode;
7883	RTGCUINTPTR GCPtrFaultAddress;
7884	uint8_t cbInstr;
7885
7886	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7887	AssertRC(rc);
7888
7889	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7890	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7891	if (enmTrpmEvent == TRPM_TRAP)
7892	{
7893	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7894	* generated using INT1 (ICEBP). */
7895	switch (uVector)
7896	{
7897	case X86_XCPT_NMI:
7898	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7899	break;
7900
7901	case X86_XCPT_BP:
7902	case X86_XCPT_OF:
7903	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7904	break;
7905
7906	case X86_XCPT_PF:
7907	case X86_XCPT_DF:
7908	case X86_XCPT_TS:
7909	case X86_XCPT_NP:
7910	case X86_XCPT_SS:
7911	case X86_XCPT_GP:
7912	case X86_XCPT_AC:
7913	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7914	RT_FALL_THRU();
7915	default:
7916	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7917	break;
7918	}
7919	}
7920	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7921	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7922	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7923	{
7924	switch (uVector)
7925	{
7926	case X86_XCPT_BP:
7927	case X86_XCPT_OF:
7928	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7929	break;
7930
7931	default:
7932	Assert(uVector == X86_XCPT_DB);
7933	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7934	break;
7935	}
7936	}
7937	else
7938	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7939
7940	rc = TRPMResetTrap(pVCpu);
7941	AssertRC(rc);
7942	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7943	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7944
7945	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7946	}
7947
7948
7949	/**
7950	* Converts the pending HM event into a TRPM trap.
7951	*
7952	* @param pVCpu The cross context virtual CPU structure.
7953	*/
7954	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7955	{
7956	Assert(pVCpu->hm.s.Event.fPending);
7957
7958	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7959	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7960	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7961	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7962
7963	/* If a trap was already pending, we did something wrong! */
7964	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7965
7966	/** @todo Use HMVmxEventToTrpmEventType() later. */
7967	TRPMEVENT enmTrapType;
7968	switch (uVectorType)
7969	{
7970	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7971	enmTrapType = TRPM_HARDWARE_INT;
7972	break;
7973
7974	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7975	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7976	enmTrapType = TRPM_TRAP;
7977	break;
7978
7979	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7980	Assert(uVector == X86_XCPT_DB);
7981	enmTrapType = TRPM_SOFTWARE_INT;
7982	break;
7983
7984	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7985	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7986	enmTrapType = TRPM_SOFTWARE_INT;
7987	break;
7988
7989	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7990	enmTrapType = TRPM_SOFTWARE_INT;
7991	break;
7992
7993	default:
7994	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7995	enmTrapType = TRPM_32BIT_HACK;
7996	break;
7997	}
7998
7999	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8000
8001	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8002	AssertRC(rc);
8003
8004	if (fErrorCodeValid)
8005	TRPMSetErrorCode(pVCpu, uErrorCode);
8006
8007	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8008	&& uVector == X86_XCPT_PF)
8009	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8010	else if (enmTrapType == TRPM_SOFTWARE_INT)
8011	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8012
8013	/* We're now done converting the pending event. */
8014	pVCpu->hm.s.Event.fPending = false;
8015	}
8016
8017
8018	/**
8019	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8020	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8021	*
8022	* @param pVCpu The cross context virtual CPU structure.
8023	* @param pVmcsInfo The VMCS info. object.
8024	*/
8025	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8026	{
8027	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8028	{
8029	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8030	{
8031	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8032	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8033	AssertRC(rc);
8034	}
8035	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8036	}
8037
8038
8039	/**
8040	* Clears the interrupt-window exiting control in the VMCS.
8041	*
8042	* @param pVmcsInfo The VMCS info. object.
8043	*/
8044	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8045	{
8046	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8047	{
8048	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8049	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8050	}
8051	return VINF_SUCCESS;
8052	}
8053
8054
8055	/**
8056	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8057	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8058	*
8059	* @param pVCpu The cross context virtual CPU structure.
8060	* @param pVmcsInfo The VMCS info. object.
8061	*/
8062	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8063	{
8064	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8065	{
8066	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8067	{
8068	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8069	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8070	AssertRC(rc);
8071	Log4Func(("Setup NMI-window exiting\n"));
8072	}
8073	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8074	}
8075
8076
8077	/**
8078	* Clears the NMI-window exiting control in the VMCS.
8079	*
8080	* @param pVmcsInfo The VMCS info. object.
8081	*/
8082	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8083	{
8084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8085	{
8086	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8087	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8088	}
8089	return VINF_SUCCESS;
8090	}
8091
8092
8093	/**
8094	* Does the necessary state syncing before returning to ring-3 for any reason
8095	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8096	*
8097	* @returns VBox status code.
8098	* @param pVCpu The cross context virtual CPU structure.
8099	* @param fImportState Whether to import the guest state from the VMCS back
8100	* to the guest-CPU context.
8101	*
8102	* @remarks No-long-jmp zone!!!
8103	*/
8104	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8105	{
8106	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8107	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8108
8109	RTCPUID idCpu = RTMpCpuId();
8110	Log4Func(("HostCpuId=%u\n", idCpu));
8111
8112	/*
8113	* !!! IMPORTANT !!!
8114	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8115	*/
8116
8117	/* Save the guest state if necessary. */
8118	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8119	if (fImportState)
8120	{
8121	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8122	AssertRCReturn(rc, rc);
8123	}
8124
8125	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8126	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8127	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8128
8129	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8130	#ifdef VBOX_STRICT
8131	if (CPUMIsHyperDebugStateActive(pVCpu))
8132	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8133	#endif
8134	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8135	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8136	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8137
8138	#if HC_ARCH_BITS == 64
8139	/* Restore host-state bits that VT-x only restores partially. */
8140	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8141	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8142	{
8143	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8144	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8145	}
8146	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8147	#endif
8148
8149	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8150	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8151	{
8152	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8153	if (!fImportState)
8154	{
8155	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8156	AssertRCReturn(rc, rc);
8157	}
8158	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8159	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8160	}
8161	else
8162	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8163
8164	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8165	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8166
8167	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8168	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8169	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8170	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8171	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8172	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8173	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8174	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8175	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8176
8177	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8178
8179	/** @todo This partially defeats the purpose of having preemption hooks.
8180	* The problem is, deregistering the hooks should be moved to a place that
8181	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8182	* context.
8183	*/
8184	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8185	AssertRCReturn(rc, rc);
8186
8187	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8188	NOREF(idCpu);
8189	return VINF_SUCCESS;
8190	}
8191
8192
8193	/**
8194	* Leaves the VT-x session.
8195	*
8196	* @returns VBox status code.
8197	* @param pVCpu The cross context virtual CPU structure.
8198	*
8199	* @remarks No-long-jmp zone!!!
8200	*/
8201	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8202	{
8203	HM_DISABLE_PREEMPT(pVCpu);
8204	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8205	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8206	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8207
8208	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8209	and done this from the VMXR0ThreadCtxCallback(). */
8210	if (!pVCpu->hm.s.fLeaveDone)
8211	{
8212	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8213	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8214	pVCpu->hm.s.fLeaveDone = true;
8215	}
8216	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8217
8218	/*
8219	* !!! IMPORTANT !!!
8220	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8221	*/
8222
8223	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8224	/** @todo Deregistering here means we need to VMCLEAR always
8225	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8226	* for calling VMMR0ThreadCtxHookDisable here! */
8227	VMMR0ThreadCtxHookDisable(pVCpu);
8228
8229	/* Leave HM context. This takes care of local init (term). */
8230	int rc = HMR0LeaveCpu(pVCpu);
8231
8232	HM_RESTORE_PREEMPT();
8233	return rc;
8234	}
8235
8236
8237	/**
8238	* Does the necessary state syncing before doing a longjmp to ring-3.
8239	*
8240	* @returns VBox status code.
8241	* @param pVCpu The cross context virtual CPU structure.
8242	*
8243	* @remarks No-long-jmp zone!!!
8244	*/
8245	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8246	{
8247	return hmR0VmxLeaveSession(pVCpu);
8248	}
8249
8250
8251	/**
8252	* Take necessary actions before going back to ring-3.
8253	*
8254	* An action requires us to go back to ring-3. This function does the necessary
8255	* steps before we can safely return to ring-3. This is not the same as longjmps
8256	* to ring-3, this is voluntary and prepares the guest so it may continue
8257	* executing outside HM (recompiler/IEM).
8258	*
8259	* @returns VBox status code.
8260	* @param pVCpu The cross context virtual CPU structure.
8261	* @param rcExit The reason for exiting to ring-3. Can be
8262	* VINF_VMM_UNKNOWN_RING3_CALL.
8263	*/
8264	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8265	{
8266	Assert(pVCpu);
8267	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8268
8269	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8270	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8271	{
8272	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8273	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8274	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8275	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8276	}
8277
8278	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8279	VMMRZCallRing3Disable(pVCpu);
8280	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8281
8282	/*
8283	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8284	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8285	*
8286	* This is because execution may continue from ring-3 and we would need to inject
8287	* the event from there (hence place it back in TRPM).
8288	*/
8289	if (pVCpu->hm.s.Event.fPending)
8290	{
8291	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8292	Assert(!pVCpu->hm.s.Event.fPending);
8293
8294	/* Clear the events from the VMCS. */
8295	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8296	AssertRCReturn(rc, rc);
8297	}
8298	#ifdef VBOX_STRICT
8299	else
8300	{
8301	/*
8302	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8303	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8304	* occasionally, see @bugref{9180#c42}.
8305	*/
8306	uint32_t uEntryIntInfo;
8307	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8308	AssertRC(rc);
8309	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8310	}
8311	#endif
8312
8313	/*
8314	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8315	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8316	* (e.g. TPR below threshold).
8317	*/
8318	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8319	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8320	AssertRCReturn(rc, rc);
8321
8322	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8323	and if we're injecting an event we should have a TRPM trap pending. */
8324	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8325	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8326	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8327	#endif
8328
8329	/* Save guest state and restore host state bits. */
8330	rc = hmR0VmxLeaveSession(pVCpu);
8331	AssertRCReturn(rc, rc);
8332	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8333
8334	/* Thread-context hooks are unregistered at this point!!! */
8335
8336	/* Sync recompiler state. */
8337	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8338	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8339	\| CPUM_CHANGED_LDTR
8340	\| CPUM_CHANGED_GDTR
8341	\| CPUM_CHANGED_IDTR
8342	\| CPUM_CHANGED_TR
8343	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8344	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8345	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8346	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8347
8348	Assert(!pVCpu->hm.s.fClearTrapFlag);
8349
8350	/* Update the exit-to-ring 3 reason. */
8351	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8352
8353	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8354	if ( rcExit != VINF_EM_RAW_INTERRUPT
8355	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8356	{
8357	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8358	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8359	}
8360
8361	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8362
8363	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8364	VMMRZCallRing3RemoveNotification(pVCpu);
8365	VMMRZCallRing3Enable(pVCpu);
8366
8367	return rc;
8368	}
8369
8370
8371	/**
8372	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8373	* longjump to ring-3 and possibly get preempted.
8374	*
8375	* @returns VBox status code.
8376	* @param pVCpu The cross context virtual CPU structure.
8377	* @param enmOperation The operation causing the ring-3 longjump.
8378	* @param pvUser User argument, currently unused, NULL.
8379	*/
8380	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8381	{
8382	RT_NOREF(pvUser);
8383	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8384	{
8385	/*
8386	* !!! IMPORTANT !!!
8387	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8388	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8389	*/
8390	VMMRZCallRing3RemoveNotification(pVCpu);
8391	VMMRZCallRing3Disable(pVCpu);
8392	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8393	RTThreadPreemptDisable(&PreemptState);
8394
8395	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8396	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8397	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8398	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8399
8400	#if HC_ARCH_BITS == 64
8401	/* Restore host-state bits that VT-x only restores partially. */
8402	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8403	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8404	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8405	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8406	#endif
8407
8408	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8409	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8410	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8411
8412	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8413	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8414	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8415
8416	/* Clear the current VMCS data back to memory. */
8417	hmR0VmxClearVmcs(pVmcsInfo);
8418
8419	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8420	VMMR0ThreadCtxHookDisable(pVCpu);
8421	HMR0LeaveCpu(pVCpu);
8422	RTThreadPreemptRestore(&PreemptState);
8423	return VINF_SUCCESS;
8424	}
8425
8426	Assert(pVCpu);
8427	Assert(pvUser);
8428	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8429	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8430
8431	VMMRZCallRing3Disable(pVCpu);
8432	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8433
8434	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8435
8436	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8437	AssertRCReturn(rc, rc);
8438
8439	VMMRZCallRing3Enable(pVCpu);
8440	return VINF_SUCCESS;
8441	}
8442
8443
8444	/**
8445	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8446	* stack.
8447	*
8448	* @returns Strict VBox status code (i.e. informational status codes too).
8449	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8450	* @param pVCpu The cross context virtual CPU structure.
8451	* @param uValue The value to push to the guest stack.
8452	*/
8453	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8454	{
8455	/*
8456	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8457	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8458	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8459	*/
8460	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8461	if (pCtx->sp == 1)
8462	return VINF_EM_RESET;
8463	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8464	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8465	AssertRC(rc);
8466	return rc;
8467	}
8468
8469
8470	/**
8471	* Injects an event into the guest upon VM-entry by updating the relevant fields
8472	* in the VM-entry area in the VMCS.
8473	*
8474	* @returns Strict VBox status code (i.e. informational status codes too).
8475	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8476	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8477	*
8478	* @param pVCpu The cross context virtual CPU structure.
8479	* @param pVmxTransient The VMX-transient structure.
8480	* @param pEvent The event being injected.
8481	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8482	* This will be updated if necessary. This cannot not
8483	* be NULL.
8484	* @param fStepping Whether we're single-stepping guest execution and
8485	* should return VINF_EM_DBG_STEPPED if the event is
8486	* injected directly (registers modified by us, not by
8487	* hardware on VM-entry).
8488	*/
8489	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8490	uint32_t *pfIntrState)
8491	{
8492	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8493	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8494	Assert(pfIntrState);
8495
8496	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8497	uint32_t u32IntInfo = pEvent->u64IntInfo;
8498	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8499	uint32_t const cbInstr = pEvent->cbInstr;
8500	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8501	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8502	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8503
8504	#ifdef VBOX_STRICT
8505	/*
8506	* Validate the error-code-valid bit for hardware exceptions.
8507	* No error codes for exceptions in real-mode.
8508	*
8509	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8510	*/
8511	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8512	&& !CPUMIsGuestInRealModeEx(pCtx))
8513	{
8514	switch (uVector)
8515	{
8516	case X86_XCPT_PF:
8517	case X86_XCPT_DF:
8518	case X86_XCPT_TS:
8519	case X86_XCPT_NP:
8520	case X86_XCPT_SS:
8521	case X86_XCPT_GP:
8522	case X86_XCPT_AC:
8523	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8524	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8525	RT_FALL_THRU();
8526	default:
8527	break;
8528	}
8529	}
8530
8531	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8532	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8533	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8534	#endif
8535
8536	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8537
8538	/*
8539	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8540	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8541	* interrupt handler in the (real-mode) guest.
8542	*
8543	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8544	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8545	*/
8546	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8547	{
8548	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8549	{
8550	/*
8551	* For CPUs with unrestricted guest execution enabled and with the guest
8552	* in real-mode, we must not set the deliver-error-code bit.
8553	*
8554	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8555	*/
8556	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8557	}
8558	else
8559	{
8560	PVM pVM = pVCpu->CTX_SUFF(pVM);
8561	Assert(PDMVmmDevHeapIsEnabled(pVM));
8562	Assert(pVM->hm.s.vmx.pRealModeTSS);
8563	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8564
8565	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8566	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8567	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8568	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8569	AssertRCReturn(rc2, rc2);
8570
8571	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8572	size_t const cbIdtEntry = sizeof(X86IDTR16);
8573	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8574	{
8575	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8576	if (uVector == X86_XCPT_DF)
8577	return VINF_EM_RESET;
8578
8579	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8580	No error codes for exceptions in real-mode. */
8581	if (uVector == X86_XCPT_GP)
8582	{
8583	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8584	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8585	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8586	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8587	HMEVENT EventXcptDf;
8588	RT_ZERO(EventXcptDf);
8589	EventXcptDf.u64IntInfo = uXcptDfInfo;
8590	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8591	}
8592
8593	/*
8594	* If we're injecting an event with no valid IDT entry, inject a #GP.
8595	* No error codes for exceptions in real-mode.
8596	*
8597	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8598	*/
8599	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8600	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8601	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8602	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8603	HMEVENT EventXcptGp;
8604	RT_ZERO(EventXcptGp);
8605	EventXcptGp.u64IntInfo = uXcptGpInfo;
8606	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8607	}
8608
8609	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8610	uint16_t uGuestIp = pCtx->ip;
8611	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8612	{
8613	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8614	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8615	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8616	}
8617	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8618	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8619
8620	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8621	X86IDTR16 IdtEntry;
8622	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8623	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8624	AssertRCReturn(rc2, rc2);
8625
8626	/* Construct the stack frame for the interrupt/exception handler. */
8627	VBOXSTRICTRC rcStrict;
8628	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8629	if (rcStrict == VINF_SUCCESS)
8630	{
8631	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8632	if (rcStrict == VINF_SUCCESS)
8633	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8634	}
8635
8636	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8637	if (rcStrict == VINF_SUCCESS)
8638	{
8639	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8640	pCtx->rip = IdtEntry.offSel;
8641	pCtx->cs.Sel = IdtEntry.uSel;
8642	pCtx->cs.ValidSel = IdtEntry.uSel;
8643	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8644	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8645	&& uVector == X86_XCPT_PF)
8646	pCtx->cr2 = GCPtrFault;
8647
8648	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8649	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8650	\| HM_CHANGED_GUEST_RSP);
8651
8652	/*
8653	* If we delivered a hardware exception (other than an NMI) and if there was
8654	* block-by-STI in effect, we should clear it.
8655	*/
8656	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8657	{
8658	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8659	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8660	Log4Func(("Clearing inhibition due to STI\n"));
8661	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8662	}
8663
8664	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8665	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8666
8667	/*
8668	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8669	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8670	*/
8671	pVCpu->hm.s.Event.fPending = false;
8672
8673	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8674	if (fStepping)
8675	rcStrict = VINF_EM_DBG_STEPPED;
8676	}
8677	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8678	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8679	return rcStrict;
8680	}
8681	}
8682
8683	/*
8684	* Validate.
8685	*/
8686	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8687	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8688
8689	/*
8690	* Inject the event into the VMCS.
8691	*/
8692	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8693	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8694	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8695	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8696	AssertRCReturn(rc, rc);
8697
8698	/*
8699	* Update guest CR2 if this is a page-fault.
8700	*/
8701	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8702	&& uVector == X86_XCPT_PF)
8703	pCtx->cr2 = GCPtrFault;
8704
8705	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8706	return VINF_SUCCESS;
8707	}
8708
8709
8710	/**
8711	* Evaluates the event to be delivered to the guest and sets it as the pending
8712	* event.
8713	*
8714	* @returns Strict VBox status code (i.e. informational status codes too).
8715	* @param pVCpu The cross context virtual CPU structure.
8716	* @param pVmxTransient The VMX-transient structure.
8717	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8718	*/
8719	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8720	{
8721	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8722	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8723
8724	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8725	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8726	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8727	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8728	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8729
8730	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8731	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8732	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8733	Assert(!TRPMHasTrap(pVCpu));
8734	Assert(pfIntrState);
8735
8736	*pfIntrState = fIntrState;
8737
8738	/*
8739	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8740	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8741	*/
8742	/** @todo SMI. SMIs take priority over NMIs. */
8743	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8744	{
8745	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8746	if ( !pVCpu->hm.s.Event.fPending
8747	&& !fBlockNmi
8748	&& !fBlockSti
8749	&& !fBlockMovSS)
8750	{
8751	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8752	if ( pVmxTransient->fIsNestedGuest
8753	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8754	return IEMExecVmxVmexitNmi(pVCpu);
8755	#endif
8756	hmR0VmxSetPendingXcptNmi(pVCpu);
8757	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8758	Log4Func(("Pending NMI\n"));
8759	}
8760	else
8761	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8762	}
8763	/*
8764	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8765	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8766	*/
8767	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8768	&& !pVCpu->hm.s.fSingleInstruction)
8769	{
8770	Assert(!DBGFIsStepping(pVCpu));
8771	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8772	AssertRCReturn(rc, rc);
8773	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8774	if ( !pVCpu->hm.s.Event.fPending
8775	&& !fBlockInt
8776	&& !fBlockSti
8777	&& !fBlockMovSS)
8778	{
8779	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8780	if ( pVmxTransient->fIsNestedGuest
8781	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8782	{
8783	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8784	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8785	return rcStrict;
8786	}
8787	#endif
8788	uint8_t u8Interrupt;
8789	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8790	if (RT_SUCCESS(rc))
8791	{
8792	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8793	if ( pVmxTransient->fIsNestedGuest
8794	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8795	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8796	{
8797	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8798	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8799	return rcStrict;
8800	}
8801	#endif
8802	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8803	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8804	}
8805	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8806	{
8807	if ( !pVmxTransient->fIsNestedGuest
8808	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8809	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8810	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8811
8812	/*
8813	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8814	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8815	* need to re-set this force-flag here.
8816	*/
8817	}
8818	else
8819	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8820	}
8821	else
8822	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8823	}
8824
8825	return VINF_SUCCESS;
8826	}
8827
8828
8829	/**
8830	* Injects any pending events into the guest if the guest is in a state to
8831	* receive them.
8832	*
8833	* @returns Strict VBox status code (i.e. informational status codes too).
8834	* @param pVCpu The cross context virtual CPU structure.
8835	* @param pVmxTransient The VMX-transient structure.
8836	* @param fIntrState The VT-x guest-interruptibility state.
8837	* @param fStepping Whether we are single-stepping the guest using the
8838	* hypervisor debugger and should return
8839	* VINF_EM_DBG_STEPPED if the event was dispatched
8840	* directly.
8841	*/
8842	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8843	{
8844	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8845	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8846
8847	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8848	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8849
8850	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8851	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8852	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8853	Assert(!TRPMHasTrap(pVCpu));
8854
8855	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8856	if (pVCpu->hm.s.Event.fPending)
8857	{
8858	/*
8859	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8860	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8861	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8862	*
8863	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8864	*/
8865	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8866	#ifdef VBOX_STRICT
8867	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8868	{
8869	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8870	Assert(!fBlockInt);
8871	Assert(!fBlockSti);
8872	Assert(!fBlockMovSS);
8873	}
8874	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8875	{
8876	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8877	Assert(!fBlockSti);
8878	Assert(!fBlockMovSS);
8879	Assert(!fBlockNmi);
8880	}
8881	#endif
8882	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8883	uIntType));
8884
8885	/*
8886	* Inject the event and get any changes to the guest-interruptibility state.
8887	*
8888	* The guest-interruptibility state may need to be updated if we inject the event
8889	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8890	*/
8891	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8892	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8893
8894	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8895	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8896	else
8897	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8898	}
8899
8900	/*
8901	* Update the guest-interruptibility state.
8902	*
8903	* This is required for the real-on-v86 software interrupt injection case above, as well as
8904	* updates to the guest state from ring-3 or IEM/REM.
8905	*/
8906	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8907	AssertRCReturn(rc, rc);
8908
8909	/*
8910	* There's no need to clear the VM-entry interruption-information field here if we're not
8911	* injecting anything. VT-x clears the valid bit on every VM-exit.
8912	*
8913	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8914	*/
8915
8916	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8917	NOREF(fBlockMovSS); NOREF(fBlockSti);
8918	return rcStrict;
8919	}
8920
8921
8922	/**
8923	* Enters the VT-x session.
8924	*
8925	* @returns VBox status code.
8926	* @param pVCpu The cross context virtual CPU structure.
8927	*/
8928	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8929	{
8930	AssertPtr(pVCpu);
8931	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8932	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8933
8934	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8935	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8936	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8937
8938	#ifdef VBOX_STRICT
8939	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8940	RTCCUINTREG uHostCR4 = ASMGetCR4();
8941	if (!(uHostCR4 & X86_CR4_VMXE))
8942	{
8943	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8944	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8945	}
8946	#endif
8947
8948	/*
8949	* Load the appropriate VMCS as the current and active one.
8950	*/
8951	PVMXVMCSINFO pVmcsInfo;
8952	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8953	if (!fInNestedGuestMode)
8954	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8955	else
8956	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8957	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8958	if (RT_SUCCESS(rc))
8959	{
8960	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8961	pVCpu->hm.s.fLeaveDone = false;
8962	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8963
8964	/*
8965	* Do the EMT scheduled L1D flush here if needed.
8966	*/
8967	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8968	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8969	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8970	hmR0MdsClear();
8971	}
8972	return rc;
8973	}
8974
8975
8976	/**
8977	* The thread-context callback (only on platforms which support it).
8978	*
8979	* @param enmEvent The thread-context event.
8980	* @param pVCpu The cross context virtual CPU structure.
8981	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8982	* @thread EMT(pVCpu)
8983	*/
8984	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8985	{
8986	NOREF(fGlobalInit);
8987
8988	switch (enmEvent)
8989	{
8990	case RTTHREADCTXEVENT_OUT:
8991	{
8992	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8993	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8994	VMCPU_ASSERT_EMT(pVCpu);
8995
8996	/* No longjmps (logger flushes, locks) in this fragile context. */
8997	VMMRZCallRing3Disable(pVCpu);
8998	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8999
9000	/* Restore host-state (FPU, debug etc.) */
9001	if (!pVCpu->hm.s.fLeaveDone)
9002	{
9003	/*
9004	* Do -not- import the guest-state here as we might already be in the middle of importing
9005	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9006	*/
9007	hmR0VmxLeave(pVCpu, false /* fImportState */);
9008	pVCpu->hm.s.fLeaveDone = true;
9009	}
9010
9011	/* Leave HM context, takes care of local init (term). */
9012	int rc = HMR0LeaveCpu(pVCpu);
9013	AssertRC(rc);
9014
9015	/* Restore longjmp state. */
9016	VMMRZCallRing3Enable(pVCpu);
9017	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9018	break;
9019	}
9020
9021	case RTTHREADCTXEVENT_IN:
9022	{
9023	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9024	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9025	VMCPU_ASSERT_EMT(pVCpu);
9026
9027	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9028	VMMRZCallRing3Disable(pVCpu);
9029	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9030
9031	/* Initialize the bare minimum state required for HM. This takes care of
9032	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9033	int rc = hmR0EnterCpu(pVCpu);
9034	AssertRC(rc);
9035	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9036	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9037
9038	/* Load the active VMCS as the current one. */
9039	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9040	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9041	AssertRC(rc);
9042	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9043	pVCpu->hm.s.fLeaveDone = false;
9044
9045	/* Do the EMT scheduled L1D flush if needed. */
9046	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9047	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9048
9049	/* Restore longjmp state. */
9050	VMMRZCallRing3Enable(pVCpu);
9051	break;
9052	}
9053
9054	default:
9055	break;
9056	}
9057	}
9058
9059
9060	/**
9061	* Exports the host state into the VMCS host-state area.
9062	* Sets up the VM-exit MSR-load area.
9063	*
9064	* The CPU state will be loaded from these fields on every successful VM-exit.
9065	*
9066	* @returns VBox status code.
9067	* @param pVCpu The cross context virtual CPU structure.
9068	*
9069	* @remarks No-long-jump zone!!!
9070	*/
9071	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9072	{
9073	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9074
9075	int rc = VINF_SUCCESS;
9076	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9077	{
9078	rc = hmR0VmxExportHostControlRegs();
9079	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9080
9081	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9082	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9083
9084	rc = hmR0VmxExportHostMsrs(pVCpu);
9085	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9086
9087	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9088	}
9089	return rc;
9090	}
9091
9092
9093	/**
9094	* Saves the host state in the VMCS host-state.
9095	*
9096	* @returns VBox status code.
9097	* @param pVCpu The cross context virtual CPU structure.
9098	*
9099	* @remarks No-long-jump zone!!!
9100	*/
9101	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9102	{
9103	AssertPtr(pVCpu);
9104	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9105
9106	/*
9107	* Export the host state here while entering HM context.
9108	* When thread-context hooks are used, we might get preempted and have to re-save the host
9109	* state but most of the time we won't be, so do it here before we disable interrupts.
9110	*/
9111	return hmR0VmxExportHostState(pVCpu);
9112	}
9113
9114
9115	/**
9116	* Exports the guest state into the VMCS guest-state area.
9117	*
9118	* The will typically be done before VM-entry when the guest-CPU state and the
9119	* VMCS state may potentially be out of sync.
9120	*
9121	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9122	* VM-entry controls.
9123	* Sets up the appropriate VMX non-root function to execute guest code based on
9124	* the guest CPU mode.
9125	*
9126	* @returns VBox strict status code.
9127	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9128	* without unrestricted guest execution and the VMMDev is not presently
9129	* mapped (e.g. EFI32).
9130	*
9131	* @param pVCpu The cross context virtual CPU structure.
9132	* @param pVmxTransient The VMX-transient structure.
9133	*
9134	* @remarks No-long-jump zone!!!
9135	*/
9136	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9137	{
9138	AssertPtr(pVCpu);
9139	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9140	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9141
9142	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9143
9144	/*
9145	* Determine real-on-v86 mode.
9146	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9147	*/
9148	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9149	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9150	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9151	pVmcsInfo->RealMode. fRealOnV86Active = false;
9152	else
9153	{
9154	Assert(!pVmxTransient->fIsNestedGuest);
9155	pVmcsInfo->RealMode.fRealOnV86Active = true;
9156	}
9157
9158	/*
9159	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9160	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9161	*/
9162	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9163	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9164	* be a need to evaluate this everytime since I'm pretty sure we intercept
9165	* all guest paging mode changes. */
9166	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9167	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9168
9169	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9170	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9171
9172	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9173	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9174
9175	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9176	if (rcStrict == VINF_SUCCESS)
9177	{ /* likely */ }
9178	else
9179	{
9180	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9181	return rcStrict;
9182	}
9183
9184	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9185	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9186
9187	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9188	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9189
9190	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9191	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9192
9193	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9194	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9195
9196	rc = hmR0VmxExportGuestRip(pVCpu);
9197	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9198	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9199	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9200
9201	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9202	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9203	\| HM_CHANGED_GUEST_CR2
9204	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9205	\| HM_CHANGED_GUEST_X87
9206	\| HM_CHANGED_GUEST_SSE_AVX
9207	\| HM_CHANGED_GUEST_OTHER_XSAVE
9208	\| HM_CHANGED_GUEST_XCRx
9209	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9210	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9211	\| HM_CHANGED_GUEST_TSC_AUX
9212	\| HM_CHANGED_GUEST_OTHER_MSRS
9213	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9214	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9215
9216	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9217	return rc;
9218	}
9219
9220
9221	/**
9222	* Exports the state shared between the host and guest into the VMCS.
9223	*
9224	* @param pVCpu The cross context virtual CPU structure.
9225	* @param pVmxTransient The VMX-transient structure.
9226	*
9227	* @remarks No-long-jump zone!!!
9228	*/
9229	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9230	{
9231	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9232	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9233
9234	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9235	{
9236	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9237	AssertRC(rc);
9238	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9239
9240	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9241	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9242	{
9243	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9244	AssertRC(rc);
9245	}
9246	}
9247
9248	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9249	{
9250	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9251	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9252	}
9253
9254	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9255	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9256	}
9257
9258
9259	/**
9260	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9261	*
9262	* @returns Strict VBox status code (i.e. informational status codes too).
9263	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9264	* without unrestricted guest execution and the VMMDev is not presently
9265	* mapped (e.g. EFI32).
9266	*
9267	* @param pVCpu The cross context virtual CPU structure.
9268	* @param pVmxTransient The VMX-transient structure.
9269	*
9270	* @remarks No-long-jump zone!!!
9271	*/
9272	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9273	{
9274	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9275	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9276	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9277
9278	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9279	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9280	#endif
9281
9282	/*
9283	* For many exits it's only RIP that changes and hence try to export it first
9284	* without going through a lot of change flag checks.
9285	*/
9286	VBOXSTRICTRC rcStrict;
9287	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9288	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9289	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9290	{
9291	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9292	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9293	{ /* likely */}
9294	else
9295	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9296	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9297	}
9298	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9299	{
9300	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9301	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9302	{ /* likely */}
9303	else
9304	{
9305	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9306	VBOXSTRICTRC_VAL(rcStrict)));
9307	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9308	return rcStrict;
9309	}
9310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9311	}
9312	else
9313	rcStrict = VINF_SUCCESS;
9314
9315	#ifdef VBOX_STRICT
9316	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9317	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9318	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9319	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9320	("fCtxChanged=%#RX64\n", fCtxChanged));
9321	#endif
9322	return rcStrict;
9323	}
9324
9325
9326	/**
9327	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9328	* and update error record fields accordingly.
9329	*
9330	* @return VMX_IGS_* return codes.
9331	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9332	* wrong with the guest state.
9333	*
9334	* @param pVCpu The cross context virtual CPU structure.
9335	* @param pVmcsInfo The VMCS info. object.
9336	*
9337	* @remarks This function assumes our cache of the VMCS controls
9338	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9339	*/
9340	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9341	{
9342	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9343	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9344	uError = (err); \
9345	break; \
9346	} else do { } while (0)
9347
9348	int rc;
9349	PVM pVM = pVCpu->CTX_SUFF(pVM);
9350	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9351	uint32_t uError = VMX_IGS_ERROR;
9352	uint32_t u32Val;
9353	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9354
9355	do
9356	{
9357	/*
9358	* CR0.
9359	*/
9360	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9361	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9362	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9363	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9364	if (fUnrestrictedGuest)
9365	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9366
9367	uint32_t u32GuestCr0;
9368	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9369	AssertRCBreak(rc);
9370	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9371	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9372	if ( !fUnrestrictedGuest
9373	&& (u32GuestCr0 & X86_CR0_PG)
9374	&& !(u32GuestCr0 & X86_CR0_PE))
9375	{
9376	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9377	}
9378
9379	/*
9380	* CR4.
9381	*/
9382	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9383	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9384
9385	uint32_t u32GuestCr4;
9386	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9387	AssertRCBreak(rc);
9388	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9389	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9390
9391	/*
9392	* IA32_DEBUGCTL MSR.
9393	*/
9394	uint64_t u64Val;
9395	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9396	AssertRCBreak(rc);
9397	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9398	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9399	{
9400	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9401	}
9402	uint64_t u64DebugCtlMsr = u64Val;
9403
9404	#ifdef VBOX_STRICT
9405	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9406	AssertRCBreak(rc);
9407	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9408	#endif
9409	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9410
9411	/*
9412	* RIP and RFLAGS.
9413	*/
9414	uint32_t u32Eflags;
9415	#if HC_ARCH_BITS == 64
9416	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9417	AssertRCBreak(rc);
9418	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9419	if ( !fLongModeGuest
9420	\|\| !pCtx->cs.Attr.n.u1Long)
9421	{
9422	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9423	}
9424	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9425	* must be identical if the "IA-32e mode guest" VM-entry
9426	* control is 1 and CS.L is 1. No check applies if the
9427	* CPU supports 64 linear-address bits. */
9428
9429	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9430	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9431	AssertRCBreak(rc);
9432	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9433	VMX_IGS_RFLAGS_RESERVED);
9434	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9435	u32Eflags = u64Val;
9436	#else
9437	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9438	AssertRCBreak(rc);
9439	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9440	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9441	#endif
9442
9443	if ( fLongModeGuest
9444	\|\| ( fUnrestrictedGuest
9445	&& !(u32GuestCr0 & X86_CR0_PE)))
9446	{
9447	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9448	}
9449
9450	uint32_t u32EntryInfo;
9451	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9452	AssertRCBreak(rc);
9453	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9454	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9455	{
9456	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9457	}
9458
9459	/*
9460	* 64-bit checks.
9461	*/
9462	#if HC_ARCH_BITS == 64
9463	if (fLongModeGuest)
9464	{
9465	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9466	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9467	}
9468
9469	if ( !fLongModeGuest
9470	&& (u32GuestCr4 & X86_CR4_PCIDE))
9471	{
9472	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9473	}
9474
9475	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9476	* 51:32 beyond the processor's physical-address width are 0. */
9477
9478	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9479	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9480	{
9481	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9482	}
9483
9484	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9485	AssertRCBreak(rc);
9486	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9487
9488	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9489	AssertRCBreak(rc);
9490	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9491	#endif
9492
9493	/*
9494	* PERF_GLOBAL MSR.
9495	*/
9496	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9497	{
9498	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9499	AssertRCBreak(rc);
9500	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9501	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9502	}
9503
9504	/*
9505	* PAT MSR.
9506	*/
9507	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9508	{
9509	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9510	AssertRCBreak(rc);
9511	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9512	for (unsigned i = 0; i < 8; i++)
9513	{
9514	uint8_t u8Val = (u64Val & 0xff);
9515	if ( u8Val != 0 /* UC */
9516	&& u8Val != 1 /* WC */
9517	&& u8Val != 4 /* WT */
9518	&& u8Val != 5 /* WP */
9519	&& u8Val != 6 /* WB */
9520	&& u8Val != 7 /* UC- */)
9521	{
9522	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9523	}
9524	u64Val >>= 8;
9525	}
9526	}
9527
9528	/*
9529	* EFER MSR.
9530	*/
9531	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9532	{
9533	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9534	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9535	AssertRCBreak(rc);
9536	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9537	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9538	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9539	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9540	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9541	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9542	* iemVmxVmentryCheckGuestState(). */
9543	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9544	\|\| !(u32GuestCr0 & X86_CR0_PG)
9545	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9546	VMX_IGS_EFER_LMA_LME_MISMATCH);
9547	}
9548
9549	/*
9550	* Segment registers.
9551	*/
9552	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9553	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9554	if (!(u32Eflags & X86_EFL_VM))
9555	{
9556	/* CS */
9557	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9558	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9559	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9560	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9561	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9562	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9563	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9564	/* CS cannot be loaded with NULL in protected mode. */
9565	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9566	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9567	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9568	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9569	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9570	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9571	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9572	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9573	else
9574	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9575
9576	/* SS */
9577	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9578	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9579	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9580	if ( !(pCtx->cr0 & X86_CR0_PE)
9581	\|\| pCtx->cs.Attr.n.u4Type == 3)
9582	{
9583	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9584	}
9585	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9586	{
9587	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9588	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9589	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9590	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9591	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9592	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9593	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9594	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9595	}
9596
9597	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9598	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9599	{
9600	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9601	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9602	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9603	\|\| pCtx->ds.Attr.n.u4Type > 11
9604	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9605	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9606	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9607	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9608	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9609	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9610	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9611	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9612	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9613	}
9614	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9615	{
9616	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9617	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9618	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9619	\|\| pCtx->es.Attr.n.u4Type > 11
9620	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9621	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9622	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9623	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9624	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9625	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9626	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9627	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9628	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9629	}
9630	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9631	{
9632	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9633	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9634	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9635	\|\| pCtx->fs.Attr.n.u4Type > 11
9636	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9637	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9638	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9639	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9640	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9641	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9642	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9643	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9644	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9645	}
9646	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9647	{
9648	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9649	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9650	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9651	\|\| pCtx->gs.Attr.n.u4Type > 11
9652	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9653	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9654	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9655	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9656	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9657	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9658	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9659	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9660	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9661	}
9662	/* 64-bit capable CPUs. */
9663	#if HC_ARCH_BITS == 64
9664	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9665	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9666	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9667	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9668	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9669	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9670	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9671	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9672	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9673	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9674	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9675	#endif
9676	}
9677	else
9678	{
9679	/* V86 mode checks. */
9680	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9681	if (pVmcsInfo->RealMode.fRealOnV86Active)
9682	{
9683	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9684	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9685	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9686	}
9687	else
9688	{
9689	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9690	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9691	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9692	}
9693
9694	/* CS */
9695	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9696	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9697	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9698	/* SS */
9699	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9700	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9701	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9702	/* DS */
9703	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9704	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9705	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9706	/* ES */
9707	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9708	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9709	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9710	/* FS */
9711	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9712	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9713	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9714	/* GS */
9715	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9716	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9717	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9718	/* 64-bit capable CPUs. */
9719	#if HC_ARCH_BITS == 64
9720	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9721	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9722	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9723	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9724	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9725	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9726	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9727	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9728	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9729	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9730	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9731	#endif
9732	}
9733
9734	/*
9735	* TR.
9736	*/
9737	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9738	/* 64-bit capable CPUs. */
9739	#if HC_ARCH_BITS == 64
9740	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9741	#endif
9742	if (fLongModeGuest)
9743	{
9744	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9745	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9746	}
9747	else
9748	{
9749	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9750	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9751	VMX_IGS_TR_ATTR_TYPE_INVALID);
9752	}
9753	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9754	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9755	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9756	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9757	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9758	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9759	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9760	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9761
9762	/*
9763	* GDTR and IDTR.
9764	*/
9765	#if HC_ARCH_BITS == 64
9766	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9767	AssertRCBreak(rc);
9768	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9769
9770	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9771	AssertRCBreak(rc);
9772	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9773	#endif
9774
9775	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9776	AssertRCBreak(rc);
9777	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9778
9779	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9780	AssertRCBreak(rc);
9781	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9782
9783	/*
9784	* Guest Non-Register State.
9785	*/
9786	/* Activity State. */
9787	uint32_t u32ActivityState;
9788	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9789	AssertRCBreak(rc);
9790	HMVMX_CHECK_BREAK( !u32ActivityState
9791	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9792	VMX_IGS_ACTIVITY_STATE_INVALID);
9793	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9794	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9795	uint32_t u32IntrState;
9796	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9797	AssertRCBreak(rc);
9798	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9799	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9800	{
9801	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9802	}
9803
9804	/** @todo Activity state and injecting interrupts. Left as a todo since we
9805	* currently don't use activity states but ACTIVE. */
9806
9807	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9808	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9809
9810	/* Guest interruptibility-state. */
9811	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9812	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9813	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9814	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9815	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9816	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9817	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9818	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9819	{
9820	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9821	{
9822	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9823	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9824	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9825	}
9826	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9827	{
9828	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9829	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9830	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9831	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9832	}
9833	}
9834	/** @todo Assumes the processor is not in SMM. */
9835	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9836	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9837	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9838	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9839	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9840	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9841	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9842	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9843	{
9844	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9845	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9846	}
9847
9848	/* Pending debug exceptions. */
9849	#if HC_ARCH_BITS == 64
9850	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9851	AssertRCBreak(rc);
9852	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9853	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9854	u32Val = u64Val; /* For pending debug exceptions checks below. */
9855	#else
9856	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9857	AssertRCBreak(rc);
9858	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9859	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9860	#endif
9861
9862	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9863	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9864	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9865	{
9866	if ( (u32Eflags & X86_EFL_TF)
9867	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9868	{
9869	/* Bit 14 is PendingDebug.BS. */
9870	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9871	}
9872	if ( !(u32Eflags & X86_EFL_TF)
9873	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9874	{
9875	/* Bit 14 is PendingDebug.BS. */
9876	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9877	}
9878	}
9879
9880	/* VMCS link pointer. */
9881	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9882	AssertRCBreak(rc);
9883	if (u64Val != UINT64_C(0xffffffffffffffff))
9884	{
9885	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9886	/** @todo Bits beyond the processor's physical-address width MBZ. */
9887	/** @todo 32-bit located in memory referenced by value of this field (as a
9888	* physical address) must contain the processor's VMCS revision ID. */
9889	/** @todo SMM checks. */
9890	}
9891
9892	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9893	* not using nested paging? */
9894	if ( pVM->hm.s.fNestedPaging
9895	&& !fLongModeGuest
9896	&& CPUMIsGuestInPAEModeEx(pCtx))
9897	{
9898	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9899	AssertRCBreak(rc);
9900	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9901
9902	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9903	AssertRCBreak(rc);
9904	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9905
9906	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9907	AssertRCBreak(rc);
9908	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9909
9910	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9911	AssertRCBreak(rc);
9912	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9913	}
9914
9915	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9916	if (uError == VMX_IGS_ERROR)
9917	uError = VMX_IGS_REASON_NOT_FOUND;
9918	} while (0);
9919
9920	pVCpu->hm.s.u32HMError = uError;
9921	return uError;
9922
9923	#undef HMVMX_ERROR_BREAK
9924	#undef HMVMX_CHECK_BREAK
9925	}
9926
9927
9928	/**
9929	* Setup the APIC-access page for virtualizing APIC access.
9930	*
9931	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9932	* this not done as part of exporting guest state, see @bugref{8721}.
9933	*
9934	* @returns VBox status code.
9935	* @param pVCpu The cross context virtual CPU structure.
9936	*/
9937	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9938	{
9939	PVM pVM = pVCpu->CTX_SUFF(pVM);
9940	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9941
9942	Assert(PDMHasApic(pVM));
9943	Assert(u64MsrApicBase);
9944
9945	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9946	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9947
9948	/* Unalias any existing mapping. */
9949	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9950	AssertRCReturn(rc, rc);
9951
9952	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9953	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9954	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9955	AssertRCReturn(rc, rc);
9956
9957	/* Update the per-VCPU cache of the APIC base MSR. */
9958	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9959	return VINF_SUCCESS;
9960	}
9961
9962
9963	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9964	/**
9965	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9966	* nested-guest using hardware-assisted VMX.
9967	*
9968	* @param pVCpu The cross context virtual CPU structure.
9969	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9970	* @param pVmcsInfoGst The guest VMCS info. object.
9971	*/
9972	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9973	{
9974	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9975	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9976	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9977	Assert(pu64MsrBitmapNstGst);
9978	Assert(pu64MsrBitmapGst);
9979	Assert(pu64MsrBitmap);
9980
9981	/*
9982	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9983	* MSR that is intercepted by the guest is also intercepted while executing the
9984	* nested-guest using hardware-assisted VMX.
9985	*/
9986	uint32_t const cbFrag = sizeof(uint64_t);
9987	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9988	for (uint32_t i = 0; i <= cFrags; i++)
9989	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9990	}
9991
9992
9993	/**
9994	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9995	* hardware-assisted VMX execution of the nested-guest.
9996	*
9997	* For a guest, we don't modify these controls once we set up the VMCS and hence
9998	* this function is never called.
9999	*
10000	* For nested-guests since the guest hypervisor provides these controls on every
10001	* nested-guest VM-entry and could potentially change them everytime we need to
10002	* merge them before every nested-guest VM-entry.
10003	*
10004	* @returns VBox status code.
10005	* @param pVCpu The cross context virtual CPU structure.
10006	*/
10007	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10008	{
10009	PVM pVM = pVCpu->CTX_SUFF(pVM);
10010	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10011	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10012	Assert(pVmcsNstGst);
10013
10014	/*
10015	* Merge the controls with the requirements of the guest VMCS.
10016	*
10017	* We do not need to validate the nested-guest VMX features specified in the
10018	* nested-guest VMCS with the features supported by the physical CPU as it's
10019	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10020	*
10021	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10022	* guest are derived from the VMX features supported by the physical CPU.
10023	*/
10024
10025	/* Pin-based VM-execution controls. */
10026	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10027
10028	/* Processor-based VM-execution controls. */
10029	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10030	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10031	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10032	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10033	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10034
10035	/* Secondary processor-based VM-execution controls. */
10036	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10037	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10038	\| VMX_PROC_CTLS2_INVPCID
10039	\| VMX_PROC_CTLS2_RDTSCP
10040	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10041	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10042	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10043	\| VMX_PROC_CTLS2_VMFUNC));
10044
10045	/*
10046	* VM-entry controls:
10047	* These controls contains state that depends on the nested-guest state (primarily
10048	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10049	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10050	* properly continue executing the nested-guest if the EFER MSR changes but does not
10051	* cause a nested-guest VM-exits.
10052	*
10053	* VM-exit controls:
10054	* These controls specify the host state on return. We cannot use the controls from
10055	* the nested-hypervisor state as is as it would contain the guest state rather than
10056	* the host state. Since the host state is subject to change (e.g. preemption, trips
10057	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10058	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10059	*
10060	* VM-entry MSR-load:
10061	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10062	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10063	*
10064	* VM-exit MSR-store:
10065	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10066	* context back into the VM-exit MSR-store area.
10067	*
10068	* VM-exit MSR-load areas:
10069	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10070	* we can entirely ignore what the nested-hypervisor wants to load here.
10071	*/
10072
10073	/*
10074	* Exception bitmap.
10075	*
10076	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10077	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10078	* keep the code more flexible if intercepting exceptions become more dynamic in
10079	* the future we do it as part of exporting the nested-guest state.
10080	*/
10081	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10082
10083	/*
10084	* CR0/CR4 guest/host mask.
10085	*
10086	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10087	* must cause VM-exits, so we need to merge them here.
10088	*/
10089	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10090	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10091
10092	/*
10093	* Page-fault error-code mask and match.
10094	*
10095	* Although we require unrestricted guest execution (and thereby nested-paging) for
10096	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10097	* normally intercept #PFs, it might intercept them for debugging purposes.
10098	*
10099	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10100	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10101	*/
10102	uint32_t u32XcptPFMask;
10103	uint32_t u32XcptPFMatch;
10104	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10105	{
10106	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10107	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10108	}
10109	else
10110	{
10111	u32XcptPFMask = 0;
10112	u32XcptPFMatch = 0;
10113	}
10114
10115	/*
10116	* Pause-Loop exiting.
10117	*/
10118	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10119	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10120
10121	/*
10122	* I/O Bitmap.
10123	*
10124	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10125	* always intercept all I/O port accesses.
10126	*/
10127	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10128
10129	/*
10130	* APIC-access page.
10131	*
10132	* The APIC-access page address has already been initialized while setting up the
10133	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10134	* should not be on any consequence to the host or to the guest for that matter, but
10135	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10136	* emulation to keep it simple.
10137	*/
10138
10139	/*
10140	* Virtual-APIC page and TPR threshold.
10141	*
10142	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10143	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10144	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10145	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10146	*/
10147	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10148	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10149	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10150	{
10151	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10152
10153	/*
10154	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10155	* we would fail to obtain a valid host-physical address for its guest-physical
10156	* address.
10157	*
10158	* We currently do not support this scenario. Maybe in the future if there is a
10159	* pressing need we can explore making this particular set of conditions work.
10160	* Right now we just cause a VM-entry failure.
10161	*
10162	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10163	* so should not really failure at the moment.
10164	*/
10165	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10166	}
10167	else
10168	{
10169	/*
10170	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10171	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10172	* be taken care of by EPT/shadow paging.
10173	*/
10174	if (pVM->hm.s.fAllow64BitGuests)
10175	{
10176	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10177	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10178	}
10179	}
10180
10181	/*
10182	* Validate basic assumptions.
10183	*/
10184	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10185	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10186	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10187	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10188
10189	/*
10190	* Commit it to the nested-guest VMCS.
10191	*/
10192	int rc = VINF_SUCCESS;
10193	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10194	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10195	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10196	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10197	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10198	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10199	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10200	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10201	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10202	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10203	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10204	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10205	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10206	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10207	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10208	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10209	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10210	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10211	{
10212	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10213	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10214	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10215	}
10216	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10217	{
10218	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10219	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10220	}
10221	AssertRCReturn(rc, rc);
10222
10223	/*
10224	* Update the nested-guest VMCS cache.
10225	*/
10226	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10227	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10228	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10229	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10230	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10231	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10232	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10233	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10234	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10235
10236	/*
10237	* MSR bitmap.
10238	*
10239	* The MSR bitmap address has already been initialized while setting up the
10240	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10241	*/
10242	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10243	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10244
10245	return VINF_SUCCESS;
10246	}
10247	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10248
10249
10250	/**
10251	* Does the preparations before executing guest code in VT-x.
10252	*
10253	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10254	* recompiler/IEM. We must be cautious what we do here regarding committing
10255	* guest-state information into the VMCS assuming we assuredly execute the
10256	* guest in VT-x mode.
10257	*
10258	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10259	* the common-state (TRPM/forceflags), we must undo those changes so that the
10260	* recompiler/IEM can (and should) use them when it resumes guest execution.
10261	* Otherwise such operations must be done when we can no longer exit to ring-3.
10262	*
10263	* @returns Strict VBox status code (i.e. informational status codes too).
10264	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10265	* have been disabled.
10266	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10267	* double-fault into the guest.
10268	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10269	* dispatched directly.
10270	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10271	*
10272	* @param pVCpu The cross context virtual CPU structure.
10273	* @param pVmxTransient The VMX-transient structure.
10274	* @param fStepping Whether we are single-stepping the guest in the
10275	* hypervisor debugger. Makes us ignore some of the reasons
10276	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10277	* if event dispatching took place.
10278	*/
10279	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10280	{
10281	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10282
10283	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10284	if (pVmxTransient->fIsNestedGuest)
10285	{
10286	RT_NOREF2(pVCpu, fStepping);
10287	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10288	return VINF_EM_RESCHEDULE_REM;
10289	}
10290	#endif
10291
10292	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10293	PGMRZDynMapFlushAutoSet(pVCpu);
10294	#endif
10295
10296	/*
10297	* Check and process force flag actions, some of which might require us to go back to ring-3.
10298	*/
10299	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10300	if (rcStrict == VINF_SUCCESS)
10301	{ /* FFs don't get set all the time. */ }
10302	else
10303	return rcStrict;
10304
10305	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10306	/*
10307	* Switch to the nested-guest VMCS as we may have transitioned into executing
10308	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10309	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10310	*
10311	* We do this as late as possible to minimize (though not completely remove)
10312	* clearing/loading VMCS again due to premature trips to ring-3 above.
10313	*/
10314	if (pVmxTransient->fIsNestedGuest)
10315	{
10316	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10317	{
10318	/*
10319	* Ensure we have synced everything from the guest VMCS and also flag that
10320	* that we need to export the full (nested) guest-CPU context to the
10321	* nested-guest VMCS.
10322	*/
10323	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10324	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10325
10326	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10327	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10328	if (RT_LIKELY(rc == VINF_SUCCESS))
10329	{
10330	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10331	ASMSetFlags(fEFlags);
10332	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10333
10334	/*
10335	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10336	* flag that we need to update the host MSR values there. Even if we decide
10337	* in the future to share the VM-exit MSR-store area page with the guest,
10338	* if its content differs, we would have to update the host MSRs anyway.
10339	*/
10340	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10341	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10342	}
10343	else
10344	{
10345	ASMSetFlags(fEFlags);
10346	return rc;
10347	}
10348	}
10349
10350	/*
10351	* Merge guest VMCS controls with the nested-guest VMCS controls.
10352	*
10353	* Even if we have not executed the guest prior to this (e.g. when resuming
10354	* from a saved state), we should be okay with merging controls as we
10355	* initialize the guest VMCS controls as part of VM setup phase.
10356	*/
10357	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10358	{
10359	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10360	AssertRCReturn(rc, rc);
10361	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10362	}
10363	}
10364	#endif
10365
10366	/*
10367	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10368	* We look at the guest VMCS control here as we always set it when supported by
10369	* the physical CPU. Looking at the nested-guest control here would not be
10370	* possible because they are not merged yet.
10371	*/
10372	PVM pVM = pVCpu->CTX_SUFF(pVM);
10373	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10374	Assert(pVmcsInfo);
10375	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10376	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10377	&& PDMHasApic(pVM))
10378	{
10379	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10380	AssertRCReturn(rc, rc);
10381	}
10382
10383	/*
10384	* Evaluate events to be injected into the guest.
10385	*
10386	* Events in TRPM can be injected without inspecting the guest state.
10387	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10388	* guest to cause a VM-exit the next time they are ready to receive the event.
10389	*/
10390	if (TRPMHasTrap(pVCpu))
10391	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10392
10393	uint32_t fIntrState;
10394	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10395
10396	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10397	/*
10398	* While evaluating pending events if something failed (unlikely) or if we were
10399	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10400	*/
10401	if ( rcStrict != VINF_SUCCESS
10402	\|\| ( pVmxTransient->fIsNestedGuest
10403	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10404	return rcStrict;
10405	#endif
10406
10407	/*
10408	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10409	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10410	* also result in triple-faulting the VM.
10411	*
10412	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10413	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10414	*/
10415	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10416	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10417	{ /* likely */ }
10418	else
10419	{
10420	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10421	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10422	return rcStrict;
10423	}
10424
10425	/*
10426	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10427	* import CR3 themselves. We will need to update them here, as even as late as the above
10428	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10429	* the below force flags to be set.
10430	*/
10431	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10432	{
10433	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10434	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10435	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10436	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10437	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10438	}
10439	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10440	{
10441	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10442	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10443	}
10444
10445	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10446	/* Paranoia. */
10447	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10448	#endif
10449
10450	/*
10451	* No longjmps to ring-3 from this point on!!!
10452	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10453	* This also disables flushing of the R0-logger instance (if any).
10454	*/
10455	VMMRZCallRing3Disable(pVCpu);
10456
10457	/*
10458	* Export the guest state bits.
10459	*
10460	* We cannot perform longjmps while loading the guest state because we do not preserve the
10461	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10462	* CPU migration.
10463	*
10464	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10465	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10466	*/
10467	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10468	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10469	{ /* likely */ }
10470	else
10471	{
10472	VMMRZCallRing3Enable(pVCpu);
10473	return rcStrict;
10474	}
10475
10476	/*
10477	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10478	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10479	* preemption disabled for a while. Since this is purely to aid the
10480	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10481	* disable interrupt on NT.
10482	*
10483	* We need to check for force-flags that could've possible been altered since we last
10484	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10485	* see @bugref{6398}).
10486	*
10487	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10488	* to ring-3 before executing guest code.
10489	*/
10490	pVmxTransient->fEFlags = ASMIntDisableFlags();
10491
10492	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10493	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10494	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10495	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10496	{
10497	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10498	{
10499	pVCpu->hm.s.Event.fPending = false;
10500
10501	/*
10502	* We've injected any pending events. This is really the point of no return (to ring-3).
10503	*
10504	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10505	* returns from this function, so don't enable them here.
10506	*/
10507	return VINF_SUCCESS;
10508	}
10509
10510	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10511	rcStrict = VINF_EM_RAW_INTERRUPT;
10512	}
10513	else
10514	{
10515	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10516	rcStrict = VINF_EM_RAW_TO_R3;
10517	}
10518
10519	ASMSetFlags(pVmxTransient->fEFlags);
10520	VMMRZCallRing3Enable(pVCpu);
10521
10522	return rcStrict;
10523	}
10524
10525
10526	/**
10527	* Final preparations before executing guest code using hardware-assisted VMX.
10528	*
10529	* We can no longer get preempted to a different host CPU and there are no returns
10530	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10531	* failures), this function is not intended to fail sans unrecoverable hardware
10532	* errors.
10533	*
10534	* @param pVCpu The cross context virtual CPU structure.
10535	* @param pVmxTransient The VMX-transient structure.
10536	*
10537	* @remarks Called with preemption disabled.
10538	* @remarks No-long-jump zone!!!
10539	*/
10540	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10541	{
10542	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10543	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10544	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10545	Assert(!pVCpu->hm.s.Event.fPending);
10546
10547	/*
10548	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10549	*/
10550	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10551	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10552
10553	PVM pVM = pVCpu->CTX_SUFF(pVM);
10554	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10555
10556	if (!CPUMIsGuestFPUStateActive(pVCpu))
10557	{
10558	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10559	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10560	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10561	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10562	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10563	}
10564
10565	/*
10566	* Re-save the host state bits as we may've been preempted (only happens when
10567	* thread-context hooks are used or when the VM start function changes).
10568	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10569	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10570	* see @bugref{8432}.
10571	*
10572	* This may also happen when switching to/from a nested-guest VMCS without leaving
10573	* ring-0.
10574	*/
10575	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10576	{
10577	int rc = hmR0VmxExportHostState(pVCpu);
10578	AssertRC(rc);
10579	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10580	}
10581	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10582
10583	/*
10584	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10585	*/
10586	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10587	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10588	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10589
10590	/*
10591	* Store status of the shared guest/host debug state at the time of VM-entry.
10592	*/
10593	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10594	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10595	{
10596	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10597	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10598	}
10599	else
10600	#endif
10601	{
10602	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10603	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10604	}
10605
10606	/*
10607	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10608	* more than one conditional check. The post-run side of our code shall determine
10609	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10610	*/
10611	if (pVmcsInfo->pbVirtApic)
10612	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10613
10614	/*
10615	* Update the host MSRs values in the VM-exit MSR-load area.
10616	*/
10617	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10618	{
10619	if (pVmcsInfo->cExitMsrLoad > 0)
10620	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10621	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10622	}
10623
10624	/*
10625	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10626	* VMX-preemption timer based on the next virtual sync clock deadline.
10627	*/
10628	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10629	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10630	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10631	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10632	{
10633	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10634	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10635	}
10636
10637	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10638	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10639	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10640	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10641
10642	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10643
10644	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10645	as we're about to start executing the guest . */
10646
10647	/*
10648	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10649	*
10650	* This is done this late as updating the TSC offsetting/preemption timer above
10651	* figures out if we can skip intercepting RDTSCP by calculating the number of
10652	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10653	*/
10654	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10655	{
10656	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10657	{
10658	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10659	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10660	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10661	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10662	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10663	AssertRC(rc);
10664	}
10665	else
10666	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10667	}
10668
10669	#ifdef VBOX_STRICT
10670	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10671	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10672	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10673	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10674	#endif
10675
10676	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10677	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10678	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10679	* see @bugref{9180#c54}. */
10680	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10681	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10682	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10683	#endif
10684	}
10685
10686
10687	/**
10688	* First C routine invoked after running guest code using hardware-assisted VMX.
10689	*
10690	* @param pVCpu The cross context virtual CPU structure.
10691	* @param pVmxTransient The VMX-transient structure.
10692	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10693	*
10694	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10695	*
10696	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10697	* unconditionally when it is safe to do so.
10698	*/
10699	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10700	{
10701	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10702
10703	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10704	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10705	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10706	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10707	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10708	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10709
10710	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10711	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10712	{
10713	uint64_t uGstTsc;
10714	if (!pVmxTransient->fIsNestedGuest)
10715	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10716	else
10717	{
10718	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10719	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10720	}
10721	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10722	}
10723
10724	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10725	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10726	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10727
10728	#if HC_ARCH_BITS == 64
10729	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10730	#endif
10731	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10732	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10733	and we need to leave it alone here. */
10734	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10735	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10736	#else
10737	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10738	#endif
10739	#ifdef VBOX_STRICT
10740	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10741	#endif
10742	Assert(!ASMIntAreEnabled());
10743	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10744	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10745
10746	/*
10747	* Save the basic VM-exit reason and check if the VM-entry failed.
10748	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10749	*/
10750	uint32_t uExitReason;
10751	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10752	AssertRC(rc);
10753	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10754	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10755
10756	/*
10757	* Check if VMLAUNCH/VMRESUME succeeded.
10758	* If this failed, we cause a guru meditation and cease further execution.
10759	*
10760	* However, if we are executing a nested-guest we might fail if we use the
10761	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10762	*/
10763	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10764	{
10765	/*
10766	* Update the VM-exit history array here even if the VM-entry failed due to:
10767	* - Invalid guest state.
10768	* - MSR loading.
10769	* - Machine-check event.
10770	*
10771	* In any of the above cases we will still have a "valid" VM-exit reason
10772	* despite @a fVMEntryFailed being false.
10773	*
10774	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10775	*
10776	* Note! We don't have CS or RIP at this point. Will probably address that later
10777	* by amending the history entry added here.
10778	*/
10779	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10780	UINT64_MAX, uHostTsc);
10781
10782	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10783	{
10784	VMMRZCallRing3Enable(pVCpu);
10785
10786	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10787	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10788
10789	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10790	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10791	AssertRC(rc);
10792	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10793	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10794	AssertRC(rc);
10795	#else
10796	/*
10797	* Import the guest-interruptibility state always as we need it while evaluating
10798	* injecting events on re-entry.
10799	*
10800	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10801	* checking for real-mode while exporting the state because all bits that cause
10802	* mode changes wrt CR0 are intercepted.
10803	*/
10804	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10805	AssertRC(rc);
10806	#endif
10807
10808	/*
10809	* Sync the TPR shadow with our APIC state.
10810	*/
10811	if ( !pVmxTransient->fIsNestedGuest
10812	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10813	{
10814	Assert(pVmcsInfo->pbVirtApic);
10815	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10816	{
10817	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10818	AssertRC(rc);
10819	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10820	}
10821	}
10822
10823	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10824	return;
10825	}
10826	}
10827	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10828	else if (pVmxTransient->fIsNestedGuest)
10829	{
10830	# if 0
10831	/*
10832	* Copy the VM-instruction error field to the guest VMCS.
10833	*/
10834	/** @todo NSTVMX: Verify we're using the fast path. */
10835	uint32_t u32RoVmInstrError;
10836	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10837	AssertRCReturn(rc, rc);
10838	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10839	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10840	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10841	# else
10842	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10843	# endif
10844	}
10845	#endif
10846	else
10847	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10848
10849	VMMRZCallRing3Enable(pVCpu);
10850	}
10851
10852
10853	/**
10854	* Runs the guest code using hardware-assisted VMX the normal way.
10855	*
10856	* @returns VBox status code.
10857	* @param pVCpu The cross context virtual CPU structure.
10858	* @param pcLoops Pointer to the number of executed loops.
10859	*/
10860	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10861	{
10862	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10863	Assert(pcLoops);
10864	Assert(*pcLoops <= cMaxResumeLoops);
10865
10866	VMXTRANSIENT VmxTransient;
10867	RT_ZERO(VmxTransient);
10868	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10869
10870	/* Paranoia. */
10871	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10872	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10873
10874	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10875	for (;;)
10876	{
10877	Assert(!HMR0SuspendPending());
10878	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10879	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10880
10881	/*
10882	* Preparatory work for running nested-guest code, this may force us to
10883	* return to ring-3.
10884	*
10885	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10886	*/
10887	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10888	if (rcStrict != VINF_SUCCESS)
10889	break;
10890
10891	/* Interrupts are disabled at this point! */
10892	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10893	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10894	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10895	/* Interrupts are re-enabled at this point! */
10896
10897	/*
10898	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10899	*/
10900	if (RT_SUCCESS(rcRun))
10901	{ /* very likely */ }
10902	else
10903	{
10904	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10905	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10906	return rcRun;
10907	}
10908
10909	/*
10910	* Profile the VM-exit.
10911	*/
10912	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10913	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10914	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10915	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10916	HMVMX_START_EXIT_DISPATCH_PROF();
10917
10918	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10919
10920	/*
10921	* Handle the VM-exit.
10922	*/
10923	#ifdef HMVMX_USE_FUNCTION_TABLE
10924	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10925	#else
10926	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10927	#endif
10928	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10929	if (rcStrict == VINF_SUCCESS)
10930	{
10931	if (++(*pcLoops) <= cMaxResumeLoops)
10932	continue;
10933	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10934	rcStrict = VINF_EM_RAW_INTERRUPT;
10935	}
10936	break;
10937	}
10938
10939	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10940	return rcStrict;
10941	}
10942
10943	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10944	/**
10945	* Runs the nested-guest code using hardware-assisted VMX.
10946	*
10947	* @returns VBox status code.
10948	* @param pVCpu The cross context virtual CPU structure.
10949	* @param pcLoops Pointer to the number of executed loops.
10950	*
10951	* @sa hmR0VmxRunGuestCodeNormal.
10952	*/
10953	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10954	{
10955	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10956	Assert(pcLoops);
10957	Assert(*pcLoops <= cMaxResumeLoops);
10958
10959	VMXTRANSIENT VmxTransient;
10960	RT_ZERO(VmxTransient);
10961	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10962	VmxTransient.fIsNestedGuest = true;
10963
10964	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10965	for (;;)
10966	{
10967	Assert(!HMR0SuspendPending());
10968	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10969	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10970
10971	/*
10972	* Preparatory work for running guest code, this may force us to
10973	* return to ring-3.
10974	*
10975	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10976	*/
10977	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10978	if (rcStrict != VINF_SUCCESS)
10979	break;
10980
10981	/* Interrupts are disabled at this point! */
10982	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10983	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10984	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10985	/* Interrupts are re-enabled at this point! */
10986
10987	/*
10988	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10989	*/
10990	if (RT_SUCCESS(rcRun))
10991	{ /* very likely */ }
10992	else
10993	{
10994	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10995	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10996	return rcRun;
10997	}
10998
10999	/*
11000	* Profile the VM-exit.
11001	*/
11002	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11004	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11005	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11006	HMVMX_START_EXIT_DISPATCH_PROF();
11007
11008	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11009
11010	/*
11011	* Handle the VM-exit.
11012	*/
11013	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11014	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11015	if ( rcStrict == VINF_SUCCESS
11016	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11017	{
11018	if (++(*pcLoops) <= cMaxResumeLoops)
11019	continue;
11020	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11021	rcStrict = VINF_EM_RAW_INTERRUPT;
11022	}
11023	break;
11024	}
11025
11026	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11027	return rcStrict;
11028	}
11029	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11030
11031
11032	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11033	* probes.
11034	*
11035	* The following few functions and associated structure contains the bloat
11036	* necessary for providing detailed debug events and dtrace probes as well as
11037	* reliable host side single stepping. This works on the principle of
11038	* "subclassing" the normal execution loop and workers. We replace the loop
11039	* method completely and override selected helpers to add necessary adjustments
11040	* to their core operation.
11041	*
11042	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11043	* any performance for debug and analysis features.
11044	*
11045	* @{
11046	*/
11047
11048	/**
11049	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11050	* the debug run loop.
11051	*/
11052	typedef struct VMXRUNDBGSTATE
11053	{
11054	/** The RIP we started executing at. This is for detecting that we stepped. */
11055	uint64_t uRipStart;
11056	/** The CS we started executing with. */
11057	uint16_t uCsStart;
11058
11059	/** Whether we've actually modified the 1st execution control field. */
11060	bool fModifiedProcCtls : 1;
11061	/** Whether we've actually modified the 2nd execution control field. */
11062	bool fModifiedProcCtls2 : 1;
11063	/** Whether we've actually modified the exception bitmap. */
11064	bool fModifiedXcptBitmap : 1;
11065
11066	/** We desire the modified the CR0 mask to be cleared. */
11067	bool fClearCr0Mask : 1;
11068	/** We desire the modified the CR4 mask to be cleared. */
11069	bool fClearCr4Mask : 1;
11070	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11071	uint32_t fCpe1Extra;
11072	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11073	uint32_t fCpe1Unwanted;
11074	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11075	uint32_t fCpe2Extra;
11076	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11077	uint32_t bmXcptExtra;
11078	/** The sequence number of the Dtrace provider settings the state was
11079	* configured against. */
11080	uint32_t uDtraceSettingsSeqNo;
11081	/** VM-exits to check (one bit per VM-exit). */
11082	uint32_t bmExitsToCheck[3];
11083
11084	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11085	uint32_t fProcCtlsInitial;
11086	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11087	uint32_t fProcCtls2Initial;
11088	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11089	uint32_t bmXcptInitial;
11090	} VMXRUNDBGSTATE;
11091	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11092	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11093
11094
11095	/**
11096	* Initializes the VMXRUNDBGSTATE structure.
11097	*
11098	* @param pVCpu The cross context virtual CPU structure of the
11099	* calling EMT.
11100	* @param pVmxTransient The VMX-transient structure.
11101	* @param pDbgState The debug state to initialize.
11102	*/
11103	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11104	{
11105	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11106	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11107
11108	pDbgState->fModifiedProcCtls = false;
11109	pDbgState->fModifiedProcCtls2 = false;
11110	pDbgState->fModifiedXcptBitmap = false;
11111	pDbgState->fClearCr0Mask = false;
11112	pDbgState->fClearCr4Mask = false;
11113	pDbgState->fCpe1Extra = 0;
11114	pDbgState->fCpe1Unwanted = 0;
11115	pDbgState->fCpe2Extra = 0;
11116	pDbgState->bmXcptExtra = 0;
11117	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11118	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11119	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11120	}
11121
11122
11123	/**
11124	* Updates the VMSC fields with changes requested by @a pDbgState.
11125	*
11126	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11127	* immediately before executing guest code, i.e. when interrupts are disabled.
11128	* We don't check status codes here as we cannot easily assert or return in the
11129	* latter case.
11130	*
11131	* @param pVCpu The cross context virtual CPU structure.
11132	* @param pVmxTransient The VMX-transient structure.
11133	* @param pDbgState The debug state.
11134	*/
11135	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11136	{
11137	/*
11138	* Ensure desired flags in VMCS control fields are set.
11139	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11140	*
11141	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11142	* there should be no stale data in pCtx at this point.
11143	*/
11144	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11145	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11146	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11147	{
11148	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11149	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11150	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11151	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11152	pDbgState->fModifiedProcCtls = true;
11153	}
11154
11155	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11156	{
11157	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11158	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11159	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11160	pDbgState->fModifiedProcCtls2 = true;
11161	}
11162
11163	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11164	{
11165	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11166	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11167	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11168	pDbgState->fModifiedXcptBitmap = true;
11169	}
11170
11171	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11172	{
11173	pVmcsInfo->u64Cr0Mask = 0;
11174	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11175	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11176	}
11177
11178	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11179	{
11180	pVmcsInfo->u64Cr4Mask = 0;
11181	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11182	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11183	}
11184
11185	NOREF(pVCpu);
11186	}
11187
11188
11189	/**
11190	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11191	* re-entry next time around.
11192	*
11193	* @returns Strict VBox status code (i.e. informational status codes too).
11194	* @param pVCpu The cross context virtual CPU structure.
11195	* @param pVmxTransient The VMX-transient structure.
11196	* @param pDbgState The debug state.
11197	* @param rcStrict The return code from executing the guest using single
11198	* stepping.
11199	*/
11200	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11201	VBOXSTRICTRC rcStrict)
11202	{
11203	/*
11204	* Restore VM-exit control settings as we may not reenter this function the
11205	* next time around.
11206	*/
11207	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11208
11209	/* We reload the initial value, trigger what we can of recalculations the
11210	next time around. From the looks of things, that's all that's required atm. */
11211	if (pDbgState->fModifiedProcCtls)
11212	{
11213	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11214	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11215	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11216	AssertRCReturn(rc2, rc2);
11217	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11218	}
11219
11220	/* We're currently the only ones messing with this one, so just restore the
11221	cached value and reload the field. */
11222	if ( pDbgState->fModifiedProcCtls2
11223	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11224	{
11225	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11226	AssertRCReturn(rc2, rc2);
11227	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11228	}
11229
11230	/* If we've modified the exception bitmap, we restore it and trigger
11231	reloading and partial recalculation the next time around. */
11232	if (pDbgState->fModifiedXcptBitmap)
11233	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11234
11235	return rcStrict;
11236	}
11237
11238
11239	/**
11240	* Configures VM-exit controls for current DBGF and DTrace settings.
11241	*
11242	* This updates @a pDbgState and the VMCS execution control fields to reflect
11243	* the necessary VM-exits demanded by DBGF and DTrace.
11244	*
11245	* @param pVCpu The cross context virtual CPU structure.
11246	* @param pVmxTransient The VMX-transient structure. May update
11247	* fUpdatedTscOffsettingAndPreemptTimer.
11248	* @param pDbgState The debug state.
11249	*/
11250	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11251	{
11252	/*
11253	* Take down the dtrace serial number so we can spot changes.
11254	*/
11255	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11256	ASMCompilerBarrier();
11257
11258	/*
11259	* We'll rebuild most of the middle block of data members (holding the
11260	* current settings) as we go along here, so start by clearing it all.
11261	*/
11262	pDbgState->bmXcptExtra = 0;
11263	pDbgState->fCpe1Extra = 0;
11264	pDbgState->fCpe1Unwanted = 0;
11265	pDbgState->fCpe2Extra = 0;
11266	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11267	pDbgState->bmExitsToCheck[i] = 0;
11268
11269	/*
11270	* Software interrupts (INT XXh) - no idea how to trigger these...
11271	*/
11272	PVM pVM = pVCpu->CTX_SUFF(pVM);
11273	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11274	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11275	{
11276	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11277	}
11278
11279	/*
11280	* INT3 breakpoints - triggered by #BP exceptions.
11281	*/
11282	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11283	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11284
11285	/*
11286	* Exception bitmap and XCPT events+probes.
11287	*/
11288	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11289	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11290	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11291
11292	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11293	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11294	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11295	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11296	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11297	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11298	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11299	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11300	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11301	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11302	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11303	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11304	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11305	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11306	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11307	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11308	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11309	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11310
11311	if (pDbgState->bmXcptExtra)
11312	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11313
11314	/*
11315	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11316	*
11317	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11318	* So, when adding/changing/removing please don't forget to update it.
11319	*
11320	* Some of the macros are picking up local variables to save horizontal space,
11321	* (being able to see it in a table is the lesser evil here).
11322	*/
11323	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11324	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11325	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11326	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11327	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11328	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11329	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11330	} else do { } while (0)
11331	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11332	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11333	{ \
11334	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11335	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11336	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11337	} else do { } while (0)
11338	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11339	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11340	{ \
11341	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11342	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11343	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11344	} else do { } while (0)
11345	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11346	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11347	{ \
11348	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11349	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11350	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11351	} else do { } while (0)
11352
11353	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11354	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11355	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11356	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11357	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11358
11359	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11360	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11361	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11362	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11363	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11364	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11365	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11366	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11367	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11368	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11369	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11370	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11371	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11372	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11373	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11374	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11375	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11376	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11377	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11378	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11379	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11380	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11381	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11382	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11383	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11384	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11385	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11386	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11387	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11388	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11389	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11390	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11391	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11392	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11393	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11394	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11395
11396	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11397	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11398	{
11399	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11400	\| CPUMCTX_EXTRN_APIC_TPR);
11401	AssertRC(rc);
11402
11403	#if 0 /** @todo fix me */
11404	pDbgState->fClearCr0Mask = true;
11405	pDbgState->fClearCr4Mask = true;
11406	#endif
11407	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11408	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11409	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11410	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11411	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11412	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11413	require clearing here and in the loop if we start using it. */
11414	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11415	}
11416	else
11417	{
11418	if (pDbgState->fClearCr0Mask)
11419	{
11420	pDbgState->fClearCr0Mask = false;
11421	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11422	}
11423	if (pDbgState->fClearCr4Mask)
11424	{
11425	pDbgState->fClearCr4Mask = false;
11426	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11427	}
11428	}
11429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11430	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11431
11432	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11433	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11434	{
11435	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11436	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11437	}
11438	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11439	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11440
11441	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11442	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11443	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11444	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11445	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11446	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11447	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11448	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11449	#if 0 /** @todo too slow, fix handler. */
11450	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11451	#endif
11452	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11453
11454	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11455	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11456	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11457	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11458	{
11459	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11460	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11461	}
11462	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11463	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11464	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11465	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11466
11467	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11468	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11469	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11470	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11471	{
11472	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11473	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11474	}
11475	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11476	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11477	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11478	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11479
11480	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11481	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11482	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11483	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11484	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11485	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11486	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11487	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11488	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11489	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11490	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11491	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11492	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11493	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11494	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11495	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11496	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11497	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11498	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11499	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11500	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11501	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11502
11503	#undef IS_EITHER_ENABLED
11504	#undef SET_ONLY_XBM_IF_EITHER_EN
11505	#undef SET_CPE1_XBM_IF_EITHER_EN
11506	#undef SET_CPEU_XBM_IF_EITHER_EN
11507	#undef SET_CPE2_XBM_IF_EITHER_EN
11508
11509	/*
11510	* Sanitize the control stuff.
11511	*/
11512	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11513	if (pDbgState->fCpe2Extra)
11514	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11515	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11516	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11517	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11518	{
11519	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11520	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11521	}
11522
11523	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11524	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11525	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11526	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11527	}
11528
11529
11530	/**
11531	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11532	* appropriate.
11533	*
11534	* The caller has checked the VM-exit against the
11535	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11536	* already, so we don't have to do that either.
11537	*
11538	* @returns Strict VBox status code (i.e. informational status codes too).
11539	* @param pVCpu The cross context virtual CPU structure.
11540	* @param pVmxTransient The VMX-transient structure.
11541	* @param uExitReason The VM-exit reason.
11542	*
11543	* @remarks The name of this function is displayed by dtrace, so keep it short
11544	* and to the point. No longer than 33 chars long, please.
11545	*/
11546	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11547	{
11548	/*
11549	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11550	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11551	*
11552	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11553	* does. Must add/change/remove both places. Same ordering, please.
11554	*
11555	* Added/removed events must also be reflected in the next section
11556	* where we dispatch dtrace events.
11557	*/
11558	bool fDtrace1 = false;
11559	bool fDtrace2 = false;
11560	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11561	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11562	uint32_t uEventArg = 0;
11563	#define SET_EXIT(a_EventSubName) \
11564	do { \
11565	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11566	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11567	} while (0)
11568	#define SET_BOTH(a_EventSubName) \
11569	do { \
11570	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11571	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11572	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11573	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11574	} while (0)
11575	switch (uExitReason)
11576	{
11577	case VMX_EXIT_MTF:
11578	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11579
11580	case VMX_EXIT_XCPT_OR_NMI:
11581	{
11582	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11583	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11584	{
11585	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11586	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11587	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11588	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11589	{
11590	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11591	{
11592	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11593	uEventArg = pVmxTransient->uExitIntErrorCode;
11594	}
11595	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11596	switch (enmEvent1)
11597	{
11598	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11599	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11600	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11601	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11602	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11603	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11604	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11605	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11606	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11607	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11608	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11609	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11610	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11611	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11612	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11613	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11614	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11615	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11616	default: break;
11617	}
11618	}
11619	else
11620	AssertFailed();
11621	break;
11622
11623	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11624	uEventArg = idxVector;
11625	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11626	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11627	break;
11628	}
11629	break;
11630	}
11631
11632	case VMX_EXIT_TRIPLE_FAULT:
11633	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11634	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11635	break;
11636	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11637	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11638	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11639	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11640	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11641
11642	/* Instruction specific VM-exits: */
11643	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11644	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11645	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11646	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11647	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11648	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11649	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11650	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11651	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11652	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11653	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11654	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11655	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11656	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11657	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11658	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11659	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11660	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11661	case VMX_EXIT_MOV_CRX:
11662	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11663	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11664	SET_BOTH(CRX_READ);
11665	else
11666	SET_BOTH(CRX_WRITE);
11667	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11668	break;
11669	case VMX_EXIT_MOV_DRX:
11670	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11671	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11672	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11673	SET_BOTH(DRX_READ);
11674	else
11675	SET_BOTH(DRX_WRITE);
11676	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11677	break;
11678	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11679	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11680	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11681	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11682	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11683	case VMX_EXIT_GDTR_IDTR_ACCESS:
11684	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11685	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11686	{
11687	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11688	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11689	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11690	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11691	}
11692	break;
11693
11694	case VMX_EXIT_LDTR_TR_ACCESS:
11695	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11696	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11697	{
11698	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11699	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11700	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11701	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11702	}
11703	break;
11704
11705	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11706	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11707	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11708	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11709	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11710	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11711	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11712	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11713	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11714	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11715	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11716
11717	/* Events that aren't relevant at this point. */
11718	case VMX_EXIT_EXT_INT:
11719	case VMX_EXIT_INT_WINDOW:
11720	case VMX_EXIT_NMI_WINDOW:
11721	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11722	case VMX_EXIT_PREEMPT_TIMER:
11723	case VMX_EXIT_IO_INSTR:
11724	break;
11725
11726	/* Errors and unexpected events. */
11727	case VMX_EXIT_INIT_SIGNAL:
11728	case VMX_EXIT_SIPI:
11729	case VMX_EXIT_IO_SMI:
11730	case VMX_EXIT_SMI:
11731	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11732	case VMX_EXIT_ERR_MSR_LOAD:
11733	case VMX_EXIT_ERR_MACHINE_CHECK:
11734	case VMX_EXIT_PML_FULL:
11735	case VMX_EXIT_VIRTUALIZED_EOI:
11736	break;
11737
11738	default:
11739	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11740	break;
11741	}
11742	#undef SET_BOTH
11743	#undef SET_EXIT
11744
11745	/*
11746	* Dtrace tracepoints go first. We do them here at once so we don't
11747	* have to copy the guest state saving and stuff a few dozen times.
11748	* Down side is that we've got to repeat the switch, though this time
11749	* we use enmEvent since the probes are a subset of what DBGF does.
11750	*/
11751	if (fDtrace1 \|\| fDtrace2)
11752	{
11753	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11754	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11755	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11756	switch (enmEvent1)
11757	{
11758	/** @todo consider which extra parameters would be helpful for each probe. */
11759	case DBGFEVENT_END: break;
11760	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11761	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11762	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11763	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11764	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11765	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11766	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11767	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11768	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11769	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11770	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11771	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11772	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11773	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11774	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11775	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11776	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11777	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11778	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11779	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11780	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11781	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11782	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11783	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11784	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11785	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11786	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11787	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11788	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11789	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11790	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11791	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11792	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11793	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11794	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11795	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11796	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11797	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11798	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11799	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11800	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11801	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11802	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11803	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11804	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11805	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11806	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11807	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11808	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11809	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11810	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11811	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11812	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11813	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11814	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11815	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11816	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11817	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11818	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11819	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11820	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11821	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11822	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11823	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11824	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11825	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11826	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11827	}
11828	switch (enmEvent2)
11829	{
11830	/** @todo consider which extra parameters would be helpful for each probe. */
11831	case DBGFEVENT_END: break;
11832	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11833	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11834	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11835	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11836	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11837	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11838	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11839	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11840	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11841	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11842	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11843	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11844	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11845	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11846	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11847	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11848	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11849	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11850	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11851	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11852	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11853	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11854	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11855	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11856	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11857	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11858	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11859	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11860	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11861	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11862	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11863	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11864	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11865	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11866	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11867	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11868	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11869	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11870	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11871	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11872	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11873	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11874	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11875	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11876	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11877	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11878	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11879	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11880	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11881	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11882	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11883	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11884	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11885	}
11886	}
11887
11888	/*
11889	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11890	* the DBGF call will do a full check).
11891	*
11892	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11893	* Note! If we have to events, we prioritize the first, i.e. the instruction
11894	* one, in order to avoid event nesting.
11895	*/
11896	PVM pVM = pVCpu->CTX_SUFF(pVM);
11897	if ( enmEvent1 != DBGFEVENT_END
11898	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11899	{
11900	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11901	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11902	if (rcStrict != VINF_SUCCESS)
11903	return rcStrict;
11904	}
11905	else if ( enmEvent2 != DBGFEVENT_END
11906	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11907	{
11908	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11909	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11910	if (rcStrict != VINF_SUCCESS)
11911	return rcStrict;
11912	}
11913
11914	return VINF_SUCCESS;
11915	}
11916
11917
11918	/**
11919	* Single-stepping VM-exit filtering.
11920	*
11921	* This is preprocessing the VM-exits and deciding whether we've gotten far
11922	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11923	* handling is performed.
11924	*
11925	* @returns Strict VBox status code (i.e. informational status codes too).
11926	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11927	* @param pVmxTransient The VMX-transient structure.
11928	* @param pDbgState The debug state.
11929	*/
11930	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11931	{
11932	/*
11933	* Expensive (saves context) generic dtrace VM-exit probe.
11934	*/
11935	uint32_t const uExitReason = pVmxTransient->uExitReason;
11936	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11937	{ /* more likely */ }
11938	else
11939	{
11940	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11941	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11942	AssertRC(rc);
11943	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11944	}
11945
11946	/*
11947	* Check for host NMI, just to get that out of the way.
11948	*/
11949	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11950	{ /* normally likely */ }
11951	else
11952	{
11953	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11954	AssertRCReturn(rc2, rc2);
11955	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11956	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11957	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11958	}
11959
11960	/*
11961	* Check for single stepping event if we're stepping.
11962	*/
11963	if (pVCpu->hm.s.fSingleInstruction)
11964	{
11965	switch (uExitReason)
11966	{
11967	case VMX_EXIT_MTF:
11968	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11969
11970	/* Various events: */
11971	case VMX_EXIT_XCPT_OR_NMI:
11972	case VMX_EXIT_EXT_INT:
11973	case VMX_EXIT_TRIPLE_FAULT:
11974	case VMX_EXIT_INT_WINDOW:
11975	case VMX_EXIT_NMI_WINDOW:
11976	case VMX_EXIT_TASK_SWITCH:
11977	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11978	case VMX_EXIT_APIC_ACCESS:
11979	case VMX_EXIT_EPT_VIOLATION:
11980	case VMX_EXIT_EPT_MISCONFIG:
11981	case VMX_EXIT_PREEMPT_TIMER:
11982
11983	/* Instruction specific VM-exits: */
11984	case VMX_EXIT_CPUID:
11985	case VMX_EXIT_GETSEC:
11986	case VMX_EXIT_HLT:
11987	case VMX_EXIT_INVD:
11988	case VMX_EXIT_INVLPG:
11989	case VMX_EXIT_RDPMC:
11990	case VMX_EXIT_RDTSC:
11991	case VMX_EXIT_RSM:
11992	case VMX_EXIT_VMCALL:
11993	case VMX_EXIT_VMCLEAR:
11994	case VMX_EXIT_VMLAUNCH:
11995	case VMX_EXIT_VMPTRLD:
11996	case VMX_EXIT_VMPTRST:
11997	case VMX_EXIT_VMREAD:
11998	case VMX_EXIT_VMRESUME:
11999	case VMX_EXIT_VMWRITE:
12000	case VMX_EXIT_VMXOFF:
12001	case VMX_EXIT_VMXON:
12002	case VMX_EXIT_MOV_CRX:
12003	case VMX_EXIT_MOV_DRX:
12004	case VMX_EXIT_IO_INSTR:
12005	case VMX_EXIT_RDMSR:
12006	case VMX_EXIT_WRMSR:
12007	case VMX_EXIT_MWAIT:
12008	case VMX_EXIT_MONITOR:
12009	case VMX_EXIT_PAUSE:
12010	case VMX_EXIT_GDTR_IDTR_ACCESS:
12011	case VMX_EXIT_LDTR_TR_ACCESS:
12012	case VMX_EXIT_INVEPT:
12013	case VMX_EXIT_RDTSCP:
12014	case VMX_EXIT_INVVPID:
12015	case VMX_EXIT_WBINVD:
12016	case VMX_EXIT_XSETBV:
12017	case VMX_EXIT_RDRAND:
12018	case VMX_EXIT_INVPCID:
12019	case VMX_EXIT_VMFUNC:
12020	case VMX_EXIT_RDSEED:
12021	case VMX_EXIT_XSAVES:
12022	case VMX_EXIT_XRSTORS:
12023	{
12024	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12025	AssertRCReturn(rc, rc);
12026	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12027	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12028	return VINF_EM_DBG_STEPPED;
12029	break;
12030	}
12031
12032	/* Errors and unexpected events: */
12033	case VMX_EXIT_INIT_SIGNAL:
12034	case VMX_EXIT_SIPI:
12035	case VMX_EXIT_IO_SMI:
12036	case VMX_EXIT_SMI:
12037	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12038	case VMX_EXIT_ERR_MSR_LOAD:
12039	case VMX_EXIT_ERR_MACHINE_CHECK:
12040	case VMX_EXIT_PML_FULL:
12041	case VMX_EXIT_VIRTUALIZED_EOI:
12042	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12043	break;
12044
12045	default:
12046	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12047	break;
12048	}
12049	}
12050
12051	/*
12052	* Check for debugger event breakpoints and dtrace probes.
12053	*/
12054	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12055	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12056	{
12057	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12058	if (rcStrict != VINF_SUCCESS)
12059	return rcStrict;
12060	}
12061
12062	/*
12063	* Normal processing.
12064	*/
12065	#ifdef HMVMX_USE_FUNCTION_TABLE
12066	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12067	#else
12068	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12069	#endif
12070	}
12071
12072
12073	/**
12074	* Single steps guest code using hardware-assisted VMX.
12075	*
12076	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12077	* but single-stepping through the hypervisor debugger.
12078	*
12079	* @returns Strict VBox status code (i.e. informational status codes too).
12080	* @param pVCpu The cross context virtual CPU structure.
12081	* @param pcLoops Pointer to the number of executed loops.
12082	*
12083	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12084	*/
12085	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12086	{
12087	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12088	Assert(pcLoops);
12089	Assert(*pcLoops <= cMaxResumeLoops);
12090
12091	VMXTRANSIENT VmxTransient;
12092	RT_ZERO(VmxTransient);
12093	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12094
12095	/* Set HMCPU indicators. */
12096	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12097	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12098	pVCpu->hm.s.fDebugWantRdTscExit = false;
12099	pVCpu->hm.s.fUsingDebugLoop = true;
12100
12101	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12102	VMXRUNDBGSTATE DbgState;
12103	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12104	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12105
12106	/*
12107	* The loop.
12108	*/
12109	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12110	for (;;)
12111	{
12112	Assert(!HMR0SuspendPending());
12113	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12114	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12115	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12116
12117	/* Set up VM-execution controls the next two can respond to. */
12118	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12119
12120	/*
12121	* Preparatory work for running guest code, this may force us to
12122	* return to ring-3.
12123	*
12124	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12125	*/
12126	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12127	if (rcStrict != VINF_SUCCESS)
12128	break;
12129
12130	/* Interrupts are disabled at this point! */
12131	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12132
12133	/* Override any obnoxious code in the above two calls. */
12134	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12135
12136	/*
12137	* Finally execute the guest.
12138	*/
12139	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12140
12141	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12142	/* Interrupts are re-enabled at this point! */
12143
12144	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12145	if (RT_SUCCESS(rcRun))
12146	{ /* very likely */ }
12147	else
12148	{
12149	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12150	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12151	return rcRun;
12152	}
12153
12154	/* Profile the VM-exit. */
12155	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12156	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12157	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12158	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12159	HMVMX_START_EXIT_DISPATCH_PROF();
12160
12161	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12162
12163	/*
12164	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12165	*/
12166	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12167	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12168	if (rcStrict != VINF_SUCCESS)
12169	break;
12170	if (++(*pcLoops) > cMaxResumeLoops)
12171	{
12172	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12173	rcStrict = VINF_EM_RAW_INTERRUPT;
12174	break;
12175	}
12176
12177	/*
12178	* Stepping: Did the RIP change, if so, consider it a single step.
12179	* Otherwise, make sure one of the TFs gets set.
12180	*/
12181	if (fStepping)
12182	{
12183	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12184	AssertRC(rc);
12185	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12186	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12187	{
12188	rcStrict = VINF_EM_DBG_STEPPED;
12189	break;
12190	}
12191	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12192	}
12193
12194	/*
12195	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12196	*/
12197	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12198	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12199	}
12200
12201	/*
12202	* Clear the X86_EFL_TF if necessary.
12203	*/
12204	if (pVCpu->hm.s.fClearTrapFlag)
12205	{
12206	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12207	AssertRC(rc);
12208	pVCpu->hm.s.fClearTrapFlag = false;
12209	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12210	}
12211	/** @todo there seems to be issues with the resume flag when the monitor trap
12212	* flag is pending without being used. Seen early in bios init when
12213	* accessing APIC page in protected mode. */
12214
12215	/*
12216	* Restore VM-exit control settings as we may not re-enter this function the
12217	* next time around.
12218	*/
12219	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12220
12221	/* Restore HMCPU indicators. */
12222	pVCpu->hm.s.fUsingDebugLoop = false;
12223	pVCpu->hm.s.fDebugWantRdTscExit = false;
12224	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12225
12226	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12227	return rcStrict;
12228	}
12229
12230
12231	/** @} */
12232
12233
12234	/**
12235	* Checks if any expensive dtrace probes are enabled and we should go to the
12236	* debug loop.
12237	*
12238	* @returns true if we should use debug loop, false if not.
12239	*/
12240	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12241	{
12242	/* It's probably faster to OR the raw 32-bit counter variables together.
12243	Since the variables are in an array and the probes are next to one
12244	another (more or less), we have good locality. So, better read
12245	eight-nine cache lines ever time and only have one conditional, than
12246	128+ conditionals, right? */
12247	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12248	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12249	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12250	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12251	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12252	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12253	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12254	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12255	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12256	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12257	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12258	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12259	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12260	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12261	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12262	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12263	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12264	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12265	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12266	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12267	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12268	) != 0
12269	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12270	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12271	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12272	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12273	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12274	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12275	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12276	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12277	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12278	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12279	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12280	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12281	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12282	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12283	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12284	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12285	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12286	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12287	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12288	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12289	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12290	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12291	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12292	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12293	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12294	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12295	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12296	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12297	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12298	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12299	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12300	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12301	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12302	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12303	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12304	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12305	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12306	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12307	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12308	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12309	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12310	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12311	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12312	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12313	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12314	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12315	) != 0
12316	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12317	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12318	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12319	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12320	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12321	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12322	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12323	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12324	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12325	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12326	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12327	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12328	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12329	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12330	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12331	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12332	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12333	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12334	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12335	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12336	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12337	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12338	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12339	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12340	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12341	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12342	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12343	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12344	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12345	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12346	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12347	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12348	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12349	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12350	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12351	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12352	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12353	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12354	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12355	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12356	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12357	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12358	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12359	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12360	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12361	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12362	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12363	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12364	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12365	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12366	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12367	) != 0;
12368	}
12369
12370
12371	/**
12372	* Runs the guest using hardware-assisted VMX.
12373	*
12374	* @returns Strict VBox status code (i.e. informational status codes too).
12375	* @param pVCpu The cross context virtual CPU structure.
12376	*/
12377	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12378	{
12379	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12380	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12381	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12382	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12383
12384	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12385
12386	VBOXSTRICTRC rcStrict;
12387	uint32_t cLoops = 0;
12388	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12389	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12390	#else
12391	bool const fInNestedGuestMode = false;
12392	#endif
12393	if (!fInNestedGuestMode)
12394	{
12395	if ( !pVCpu->hm.s.fUseDebugLoop
12396	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12397	&& !DBGFIsStepping(pVCpu)
12398	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12399	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12400	else
12401	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12402	}
12403	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12404	else
12405	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12406
12407	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12408	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12409	#endif
12410
12411	if (rcStrict == VERR_EM_INTERPRETER)
12412	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12413	else if (rcStrict == VINF_EM_RESET)
12414	rcStrict = VINF_EM_TRIPLE_FAULT;
12415
12416	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12417	if (RT_FAILURE(rc2))
12418	{
12419	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12420	rcStrict = rc2;
12421	}
12422	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12423	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12424	return rcStrict;
12425	}
12426
12427
12428	#ifndef HMVMX_USE_FUNCTION_TABLE
12429	/**
12430	* Handles a guest VM-exit from hardware-assisted VMX execution.
12431	*
12432	* @returns Strict VBox status code (i.e. informational status codes too).
12433	* @param pVCpu The cross context virtual CPU structure.
12434	* @param pVmxTransient The VMX-transient structure.
12435	*/
12436	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12437	{
12438	#ifdef DEBUG_ramshankar
12439	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12440	do { \
12441	if (a_fSave != 0) \
12442	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12443	VBOXSTRICTRC rcStrict = a_CallExpr; \
12444	if (a_fSave != 0) \
12445	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12446	return rcStrict; \
12447	} while (0)
12448	#else
12449	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12450	#endif
12451	uint32_t const rcReason = pVmxTransient->uExitReason;
12452	switch (rcReason)
12453	{
12454	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12455	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12456	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12457	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12458	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12459	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12460	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12461	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12462	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12463	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12464	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12465	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12466	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12467	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12468	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12469	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12470	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12471	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12472	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12473	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12474	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12475	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12476	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12477	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12478	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12479	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12480	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12481	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12482	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12483	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12484	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12485	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12486	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12487	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12488	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12489	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12490	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12491	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12492	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12493	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12494	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12495	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12496	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12497	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12498	#else
12499	case VMX_EXIT_VMCLEAR:
12500	case VMX_EXIT_VMLAUNCH:
12501	case VMX_EXIT_VMPTRLD:
12502	case VMX_EXIT_VMPTRST:
12503	case VMX_EXIT_VMREAD:
12504	case VMX_EXIT_VMRESUME:
12505	case VMX_EXIT_VMWRITE:
12506	case VMX_EXIT_VMXOFF:
12507	case VMX_EXIT_VMXON:
12508	case VMX_EXIT_INVVPID:
12509	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12510	#endif
12511
12512	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12513	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12514	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12515
12516	case VMX_EXIT_RSM:
12517	case VMX_EXIT_RDSEED:
12518	case VMX_EXIT_ENCLS:
12519	case VMX_EXIT_INVEPT:
12520	case VMX_EXIT_VMFUNC:
12521	case VMX_EXIT_XSAVES:
12522	case VMX_EXIT_XRSTORS:
12523	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12524
12525	case VMX_EXIT_INIT_SIGNAL:
12526	case VMX_EXIT_SIPI:
12527	case VMX_EXIT_IO_SMI:
12528	case VMX_EXIT_SMI:
12529	case VMX_EXIT_ERR_MSR_LOAD:
12530	case VMX_EXIT_ERR_MACHINE_CHECK:
12531	case VMX_EXIT_PML_FULL:
12532	case VMX_EXIT_VIRTUALIZED_EOI:
12533	case VMX_EXIT_APIC_WRITE:
12534	default:
12535	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12536	}
12537	#undef VMEXIT_CALL_RET
12538	}
12539	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12540
12541
12542	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12543	/**
12544	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12545	*
12546	* @returns Strict VBox status code (i.e. informational status codes too).
12547	* @param pVCpu The cross context virtual CPU structure.
12548	* @param pVmxTransient The VMX-transient structure.
12549	*/
12550	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12551	{
12552	VBOXSTRICTRC rcStrict;
12553	uint32_t const uExitReason = pVmxTransient->uExitReason;
12554	switch (uExitReason)
12555	{
12556	case VMX_EXIT_EPT_MISCONFIG:
12557	rcStrict = hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12558	break;
12559
12560	case VMX_EXIT_EPT_VIOLATION:
12561	rcStrict = hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12562	break;
12563
12564	case VMX_EXIT_IO_INSTR:
12565	{
12566	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12567	AssertRCReturn(rc, rc);
12568
12569	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12570	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12571	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12572
12573	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
12574	uint8_t const cbAccess = s_aIOSizes[uIOSize];
12575	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
12576	{
12577	/*
12578	* IN/OUT instruction:
12579	* - Provides VM-exit instruction length.
12580	*
12581	* INS/OUTS instruction:
12582	* - Provides VM-exit instruction length.
12583	* - Provides Guest-linear address.
12584	* - Optionally provides VM-exit instruction info (depends on CPU feature).
12585	*/
12586	PVM pVM = pVCpu->CTX_SUFF(pVM);
12587	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12588	AssertRCReturn(rc, rc);
12589
12590	/* Make sure we don't use stale VMX-transient info. */
12591	pVmxTransient->ExitInstrInfo.u = 0;
12592	pVmxTransient->uGuestLinearAddr = 0;
12593
12594	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
12595	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12596	if (fIOString)
12597	{
12598	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12599	if (fVmxInsOutsInfo)
12600	{
12601	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
12602	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12603	}
12604	}
12605	AssertRCReturn(rc, rc);
12606
12607	VMXVEXITINFO ExitInfo;
12608	RT_ZERO(ExitInfo);
12609	ExitInfo.uReason = uExitReason;
12610	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12611	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12612	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12613	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
12614	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12615	}
12616	else
12617	rcStrict = hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
12618	break;
12619	}
12620
12621	case VMX_EXIT_HLT:
12622	{
12623	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12624	AssertRCReturn(rc, rc);
12625	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
12626	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12627	else
12628	rcStrict = hmR0VmxExitHlt(pVCpu, pVmxTransient);
12629	break;
12630	}
12631
12632	case VMX_EXIT_RDTSC:
12633	{
12634	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12635	AssertRCReturn(rc, rc);
12636	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12637	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12638	else
12639	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12640	break;
12641	}
12642
12643	case VMX_EXIT_RDTSCP:
12644	{
12645	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12646	AssertRCReturn(rc, rc);
12647	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12648	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12649	else
12650	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12651	break;
12652	}
12653
12654	/*
12655	* Instructions that cause VM-exits unconditionally.
12656	* - Provides VM-exit instruction length ONLY.
12657	*/
12658	case VMX_EXIT_CPUID:
12659	case VMX_EXIT_VMCALL:
12660	case VMX_EXIT_GETSEC:
12661	case VMX_EXIT_INVD:
12662	case VMX_EXIT_XSETBV:
12663	case VMX_EXIT_VMLAUNCH:
12664	case VMX_EXIT_VMRESUME:
12665	case VMX_EXIT_VMXOFF:
12666	{
12667	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12668	AssertRCReturn(rc, rc);
12669	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12670	break;
12671	}
12672
12673	/*
12674	* Instructions that cause VM-exits unconditionally.
12675	* - Provides VM-exit instruction length.
12676	* - Provides VM-exit information.
12677	* - Optionally provides VM-exit qualification.
12678	*
12679	* Since VM-exit qualification is 0 for all VM-exits where it is not
12680	* applicable, reading and passing it to the guest should produce
12681	* defined behavior.
12682	*
12683	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12684	*/
12685	case VMX_EXIT_INVEPT:
12686	case VMX_EXIT_INVVPID:
12687	case VMX_EXIT_VMCLEAR:
12688	case VMX_EXIT_VMPTRLD:
12689	case VMX_EXIT_VMPTRST:
12690	case VMX_EXIT_VMXON:
12691	{
12692	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12693	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12694	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12695	AssertRCReturn(rc, rc);
12696
12697	VMXVEXITINFO ExitInfo;
12698	RT_ZERO(ExitInfo);
12699	ExitInfo.uReason = uExitReason;
12700	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12701	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12702	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12703	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12704	break;
12705	}
12706
12707	case VMX_EXIT_INVLPG:
12708	{
12709	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12710	{
12711	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12712	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12713	AssertRCReturn(rc, rc);
12714
12715	VMXVEXITINFO ExitInfo;
12716	RT_ZERO(ExitInfo);
12717	ExitInfo.uReason = uExitReason;
12718	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12719	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12720	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12721	}
12722	else
12723	rcStrict = hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
12724	break;
12725	}
12726
12727	case VMX_EXIT_INVPCID:
12728	{
12729	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12730	{
12731	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12732	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12733	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12734	AssertRCReturn(rc, rc);
12735
12736	VMXVEXITINFO ExitInfo;
12737	RT_ZERO(ExitInfo);
12738	ExitInfo.uReason = uExitReason;
12739	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12740	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12741	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12742	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12743	}
12744	else
12745	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12746	break;
12747	}
12748
12749	case VMX_EXIT_RDMSR:
12750	{
12751	uint32_t fMsrpm;
12752	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12753	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12754	else
12755	fMsrpm = VMXMSRPM_EXIT_RD;
12756
12757	if (fMsrpm & VMXMSRPM_EXIT_RD)
12758	{
12759	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12760	AssertRCReturn(rc, rc);
12761	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12762	}
12763	else
12764	rcStrict = hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
12765	break;
12766	}
12767
12768	case VMX_EXIT_WRMSR:
12769	{
12770	uint32_t fMsrpm;
12771	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12772	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12773	else
12774	fMsrpm = VMXMSRPM_EXIT_WR;
12775
12776	if (fMsrpm & VMXMSRPM_EXIT_WR)
12777	{
12778	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12779	AssertRCReturn(rc, rc);
12780	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12781	}
12782	else
12783	rcStrict = hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
12784	break;
12785	}
12786
12787	case VMX_EXIT_TASK_SWITCH:
12788	{
12789	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12790	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12791	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12792	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12793	AssertRCReturn(rc, rc);
12794
12795	VMXVEXITINFO ExitInfo;
12796	RT_ZERO(ExitInfo);
12797	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12798	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12799
12800	VMXVEXITEVENTINFO ExitEventInfo;
12801	RT_ZERO(ExitInfo);
12802	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
12803	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
12804
12805	rcStrict = IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
12806	break;
12807	}
12808
12809	case VMX_EXIT_WBINVD:
12810	{
12811	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
12812	{
12813	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12814	AssertRCReturn(rc, rc);
12815	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12816	}
12817	else
12818	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12819	break;
12820	}
12821
12822	case VMX_EXIT_MTF:
12823	{
12824	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
12825	rcStrict = IEMExecVmxVmexit(pVCpu, uExitReason);
12826	break;
12827	}
12828
12829	case VMX_EXIT_APIC_ACCESS:
12830	case VMX_EXIT_XCPT_OR_NMI:
12831	{
12832	/** @todo NSTVMX: APIC-access, Xcpt or NMI, Mov CRx. */
12833	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12834	break;
12835	}
12836
12837	case VMX_EXIT_MOV_CRX:
12838	{
12839	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12840	AssertRCReturn(rc, rc);
12841
12842	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
12843	switch (uAccessType)
12844	{
12845	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
12846	{
12847	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
12848	Assert(pVmcsNstGst);
12849	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12850	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
12851	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
12852	uint64_t const uNewCrx = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
12853	if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrx))
12854	{
12855	VMXVEXITINFO ExitInfo;
12856	RT_ZERO(ExitInfo);
12857	ExitInfo.uReason = VMX_EXIT_MOV_CRX;
12858	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12859	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12860	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12861	}
12862	else
12863	rcStrict = hmR0VmxExitMovCRx(pVCpu, pVmxTransient);
12864	break;
12865	}
12866
12867	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
12868	{
12869	/** @todo NSTVMX: Implement me. */
12870	break;
12871	}
12872
12873	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
12874	{
12875	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12876	AssertRCReturn(rc, rc);
12877
12878	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
12879	Assert(pVmcsNstGst);
12880	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
12881	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
12882	if ( (uGstHostMask & X86_CR0_TS)
12883	&& (uReadShadow & X86_CR0_TS))
12884	{
12885	VMXVEXITINFO ExitInfo;
12886	RT_ZERO(ExitInfo);
12887	ExitInfo.uReason = VMX_EXIT_MOV_CRX;
12888	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12889	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12890	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12891	}
12892	else
12893	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->cbInstr);
12894	break;
12895	}
12896
12897	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12898	{
12899	RTGCPTR GCPtrEffDst;
12900	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
12901	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
12902
12903	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12904	if (fMemOperand)
12905	{
12906	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12907	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
12908	}
12909	else
12910	GCPtrEffDst = NIL_RTGCPTR;
12911	AssertRCReturn(rc, rc);
12912
12913	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
12914	{
12915	VMXVEXITINFO ExitInfo;
12916	RT_ZERO(ExitInfo);
12917	ExitInfo.uReason = VMX_EXIT_MOV_CRX;
12918	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12919	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
12920	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12921	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12922	}
12923	else
12924	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
12925	break;
12926	}
12927
12928	default:
12929	{
12930	pVCpu->hm.s.u32HMError = uAccessType;
12931	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12932	VERR_VMX_UNEXPECTED_EXCEPTION);
12933	}
12934	}
12935	break;
12936	}
12937
12938	case VMX_EXIT_EXT_INT:
12939	{
12940	/* We shouldn't direct physical interrupts to the nested-guest. */
12941	rcStrict = hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12942	break;
12943	}
12944
12945	case VMX_EXIT_INT_WINDOW:
12946	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12947	{
12948	/** @todo NSTVMX: Interrupt window, TPR below threshold. */
12949	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12950	break;
12951	}
12952
12953	case VMX_EXIT_MWAIT:
12954	{
12955	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
12956	{
12957	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12958	AssertRCReturn(rc, rc);
12959	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12960	}
12961	else
12962	rcStrict = hmR0VmxExitMwait(pVCpu, pVmxTransient);
12963	break;
12964	}
12965
12966	case VMX_EXIT_MONITOR:
12967	{
12968	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
12969	{
12970	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12971	AssertRCReturn(rc, rc);
12972	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12973	}
12974	else
12975	rcStrict = hmR0VmxExitMonitor(pVCpu, pVmxTransient);
12976	break;
12977	}
12978
12979	case VMX_EXIT_PAUSE:
12980	{
12981	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
12982	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
12983	* PAUSE when executing a nested-guest? If it does not, we would not need
12984	* to check for the intercepts here. Just call VM-exit... */
12985	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
12986	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
12987	{
12988	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12989	AssertRCReturn(rc, rc);
12990	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12991	}
12992	else
12993	rcStrict = hmR0VmxExitPause(pVCpu, pVmxTransient);
12994	break;
12995	}
12996
12997	case VMX_EXIT_PREEMPT_TIMER:
12998	{
12999	/** @todo NSTVMX: Preempt timer. */
13000	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13001	break;
13002	}
13003
13004	case VMX_EXIT_MOV_DRX:
13005	{
13006	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
13007	{
13008	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13009	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13010	AssertRCReturn(rc, rc);
13011
13012	VMXVEXITINFO ExitInfo;
13013	RT_ZERO(ExitInfo);
13014	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13015	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13016	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13017	}
13018	else
13019	rcStrict = hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
13020	break;
13021	}
13022
13023	case VMX_EXIT_GDTR_IDTR_ACCESS:
13024	case VMX_EXIT_LDTR_TR_ACCESS:
13025	{
13026	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
13027	{
13028	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13029	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13030	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13031	AssertRCReturn(rc, rc);
13032
13033	VMXVEXITINFO ExitInfo;
13034	RT_ZERO(ExitInfo);
13035	ExitInfo.cbInstr = pVmxTransient->cbInstr;
13036	ExitInfo.u64Qual = pVmxTransient->uExitQual;
13037	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
13038	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
13039	}
13040	else
13041	rcStrict = hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient);
13042	break;
13043	}
13044
13045	case VMX_EXIT_RDRAND:
13046	case VMX_EXIT_RDPMC:
13047	case VMX_EXIT_VMREAD:
13048	case VMX_EXIT_VMWRITE:
13049	case VMX_EXIT_RSM:
13050	case VMX_EXIT_RDSEED:
13051	case VMX_EXIT_ENCLS:
13052	case VMX_EXIT_VMFUNC:
13053	case VMX_EXIT_XSAVES:
13054	case VMX_EXIT_XRSTORS:
13055
13056	case VMX_EXIT_TRIPLE_FAULT:
13057	case VMX_EXIT_NMI_WINDOW:
13058	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
13059
13060	case VMX_EXIT_INIT_SIGNAL:
13061	case VMX_EXIT_SIPI:
13062	case VMX_EXIT_IO_SMI:
13063	case VMX_EXIT_SMI:
13064	case VMX_EXIT_ERR_MSR_LOAD:
13065	case VMX_EXIT_ERR_MACHINE_CHECK:
13066	case VMX_EXIT_PML_FULL:
13067	case VMX_EXIT_VIRTUALIZED_EOI:
13068	case VMX_EXIT_APIC_WRITE:
13069	default:
13070	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13071	break;
13072	}
13073
13074	return rcStrict;
13075	}
13076	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13077
13078
13079	#ifdef VBOX_STRICT
13080	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13081	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13082	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13083
13084	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13085	do { \
13086	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13087	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13088	} while (0)
13089
13090	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13091	do { \
13092	AssertPtr((a_pVCpu)); \
13093	AssertPtr((a_pVmxTransient)); \
13094	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13095	Assert((a_pVmxTransient)->pVmcsInfo); \
13096	Assert(ASMIntAreEnabled()); \
13097	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13098	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13099	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
13100	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13101	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13102	HMVMX_ASSERT_PREEMPT_CPUID(); \
13103	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13104	} while (0)
13105
13106	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13107	do { \
13108	Log4Func(("\n")); \
13109	} while (0)
13110	#else
13111	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13112	do { \
13113	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13114	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13115	} while (0)
13116	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13117	#endif
13118
13119
13120	/**
13121	* Advances the guest RIP by the specified number of bytes.
13122	*
13123	* @param pVCpu The cross context virtual CPU structure.
13124	* @param cbInstr Number of bytes to advance the RIP by.
13125	*
13126	* @remarks No-long-jump zone!!!
13127	*/
13128	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13129	{
13130	/* Advance the RIP. */
13131	pVCpu->cpum.GstCtx.rip += cbInstr;
13132	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13133
13134	/* Update interrupt inhibition. */
13135	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13136	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13137	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13138	}
13139
13140
13141	/**
13142	* Advances the guest RIP after reading it from the VMCS.
13143	*
13144	* @returns VBox status code, no informational status codes.
13145	* @param pVCpu The cross context virtual CPU structure.
13146	* @param pVmxTransient The VMX-transient structure.
13147	*
13148	* @remarks No-long-jump zone!!!
13149	*/
13150	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13151	{
13152	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13153	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13154	AssertRCReturn(rc, rc);
13155
13156	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13157	return VINF_SUCCESS;
13158	}
13159
13160
13161	/**
13162	* Handle a condition that occurred while delivering an event through the guest
13163	* IDT.
13164	*
13165	* @returns Strict VBox status code (i.e. informational status codes too).
13166	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13167	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13168	* to continue execution of the guest which will delivery the \#DF.
13169	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13170	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13171	*
13172	* @param pVCpu The cross context virtual CPU structure.
13173	* @param pVmxTransient The VMX-transient structure.
13174	*
13175	* @remarks No-long-jump zone!!!
13176	*/
13177	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13178	{
13179	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13180
13181	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13182	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13183	AssertRCReturn(rc2, rc2);
13184
13185	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13186	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13187	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13188	{
13189	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13190	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13191
13192	/*
13193	* If the event was a software interrupt (generated with INT n) or a software exception
13194	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13195	* can handle the VM-exit and continue guest execution which will re-execute the
13196	* instruction rather than re-injecting the exception, as that can cause premature
13197	* trips to ring-3 before injection and involve TRPM which currently has no way of
13198	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13199	* the problem).
13200	*/
13201	IEMXCPTRAISE enmRaise;
13202	IEMXCPTRAISEINFO fRaiseInfo;
13203	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13204	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13205	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13206	{
13207	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13208	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13209	}
13210	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13211	{
13212	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13213	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13214	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13215	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13216	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13217	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13218
13219	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13220
13221	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13222	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13223	{
13224	pVmxTransient->fVectoringPF = true;
13225	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13226	}
13227	}
13228	else
13229	{
13230	/*
13231	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13232	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13233	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13234	*/
13235	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13236	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13237	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13238	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13239	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13240	}
13241
13242	/*
13243	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13244	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13245	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13246	* subsequent VM-entry would fail.
13247	*
13248	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13249	*/
13250	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
13251	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13252	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13253	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13254	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13255	{
13256	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
13257	}
13258
13259	switch (enmRaise)
13260	{
13261	case IEMXCPTRAISE_CURRENT_XCPT:
13262	{
13263	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13264	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13265	Assert(rcStrict == VINF_SUCCESS);
13266	break;
13267	}
13268
13269	case IEMXCPTRAISE_PREV_EVENT:
13270	{
13271	uint32_t u32ErrCode;
13272	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13273	{
13274	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13275	AssertRCReturn(rc2, rc2);
13276	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13277	}
13278	else
13279	u32ErrCode = 0;
13280
13281	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13282	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13283	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13284	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13285
13286	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13287	pVCpu->hm.s.Event.u32ErrCode));
13288	Assert(rcStrict == VINF_SUCCESS);
13289	break;
13290	}
13291
13292	case IEMXCPTRAISE_REEXEC_INSTR:
13293	Assert(rcStrict == VINF_SUCCESS);
13294	break;
13295
13296	case IEMXCPTRAISE_DOUBLE_FAULT:
13297	{
13298	/*
13299	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13300	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13301	*/
13302	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13303	{
13304	pVmxTransient->fVectoringDoublePF = true;
13305	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13306	pVCpu->cpum.GstCtx.cr2));
13307	rcStrict = VINF_SUCCESS;
13308	}
13309	else
13310	{
13311	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13312	hmR0VmxSetPendingXcptDF(pVCpu);
13313	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13314	uIdtVector, uExitVector));
13315	rcStrict = VINF_HM_DOUBLE_FAULT;
13316	}
13317	break;
13318	}
13319
13320	case IEMXCPTRAISE_TRIPLE_FAULT:
13321	{
13322	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13323	rcStrict = VINF_EM_RESET;
13324	break;
13325	}
13326
13327	case IEMXCPTRAISE_CPU_HANG:
13328	{
13329	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13330	rcStrict = VERR_EM_GUEST_CPU_HANG;
13331	break;
13332	}
13333
13334	default:
13335	{
13336	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13337	rcStrict = VERR_VMX_IPE_2;
13338	break;
13339	}
13340	}
13341	}
13342	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13343	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13344	&& uExitVector != X86_XCPT_DF
13345	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13346	{
13347	/*
13348	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13349	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13350	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13351	*/
13352	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
13353	{
13354	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
13355	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
13356	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
13357	}
13358	}
13359
13360	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13361	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13362	return rcStrict;
13363	}
13364
13365
13366	/** @name VM-exit handlers.
13367	* @{
13368	*/
13369	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13370	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13371	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13372
13373	/**
13374	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13375	*/
13376	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13377	{
13378	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13380	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13381	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13382	return VINF_SUCCESS;
13383	return VINF_EM_RAW_INTERRUPT;
13384	}
13385
13386
13387	/**
13388	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13389	*/
13390	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13391	{
13392	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13393	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13394
13395	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13396	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13397	AssertRCReturn(rc, rc);
13398
13399	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13400	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13401	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13402	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13403
13404	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13405	{
13406	/*
13407	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13408	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13409	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13410	*
13411	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13412	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13413	*/
13414	VMXDispatchHostNmi();
13415	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
13416	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13417	return VINF_SUCCESS;
13418	}
13419
13420	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13421	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13422	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
13423	{ /* likely */ }
13424	else
13425	{
13426	if (rcStrict == VINF_HM_DOUBLE_FAULT)
13427	rcStrict = VINF_SUCCESS;
13428	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13429	return rcStrict;
13430	}
13431
13432	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13433	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13434	switch (uIntType)
13435	{
13436	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13437	Assert(uVector == X86_XCPT_DB);
13438	RT_FALL_THRU();
13439	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13440	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13441	RT_FALL_THRU();
13442	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13443	{
13444	/*
13445	* If there's any exception caused as a result of event injection, the resulting
13446	* secondary/final execption will be pending, we shall continue guest execution
13447	* after injecting the event. The page-fault case is complicated and we manually
13448	* handle any currently pending event in hmR0VmxExitXcptPF.
13449	*/
13450	if (!pVCpu->hm.s.Event.fPending)
13451	{ /* likely */ }
13452	else if (uVector != X86_XCPT_PF)
13453	{
13454	rcStrict = VINF_SUCCESS;
13455	break;
13456	}
13457
13458	switch (uVector)
13459	{
13460	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13461	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13462	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13463	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13464	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13465	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13466
13467	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13468	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13469	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13470	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13471	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13472	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13473	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13474	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13475	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13476	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13477	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13478	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13479	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13480	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13481	default:
13482	{
13483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13484	if (pVmcsInfo->RealMode.fRealOnV86Active)
13485	{
13486	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13487	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13488	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13489
13490	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13491	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13492	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13493	AssertRCReturn(rc, rc);
13494	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13495	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13496	0 /* GCPtrFaultAddress */);
13497	rcStrict = VINF_SUCCESS;
13498	}
13499	else
13500	{
13501	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13502	pVCpu->hm.s.u32HMError = uVector;
13503	rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
13504	}
13505	break;
13506	}
13507	}
13508	break;
13509	}
13510
13511	default:
13512	{
13513	pVCpu->hm.s.u32HMError = uExitIntInfo;
13514	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13515	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13516	break;
13517	}
13518	}
13519	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13520	return rcStrict;
13521	}
13522
13523
13524	/**
13525	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13526	*/
13527	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13528	{
13529	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13530
13531	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13532	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13533	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13534	AssertRCReturn(rc, rc);
13535
13536	/* Evaluate and deliver pending events and resume guest execution. */
13537	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13538	return VINF_SUCCESS;
13539	}
13540
13541
13542	/**
13543	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13544	*/
13545	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13546	{
13547	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13548
13549	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13550	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13551	{
13552	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13553	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13554	}
13555
13556	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13557
13558	/*
13559	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13560	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13561	*/
13562	uint32_t fIntrState;
13563	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13564	AssertRCReturn(rc, rc);
13565	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13566	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13567	{
13568	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13569	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13570
13571	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13572	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13573	AssertRCReturn(rc, rc);
13574	}
13575
13576	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13577	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13578	AssertRCReturn(rc, rc);
13579
13580	/* Evaluate and deliver pending events and resume guest execution. */
13581	return VINF_SUCCESS;
13582	}
13583
13584
13585	/**
13586	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13587	*/
13588	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13589	{
13590	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13591	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13592	}
13593
13594
13595	/**
13596	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13597	*/
13598	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13599	{
13600	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13601	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13602	}
13603
13604
13605	/**
13606	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13607	*/
13608	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13609	{
13610	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13611
13612	/*
13613	* Get the state we need and update the exit history entry.
13614	*/
13615	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13616	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13617	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13618	AssertRCReturn(rc, rc);
13619
13620	VBOXSTRICTRC rcStrict;
13621	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13622	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13623	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13624	if (!pExitRec)
13625	{
13626	/*
13627	* Regular CPUID instruction execution.
13628	*/
13629	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13630	if (rcStrict == VINF_SUCCESS)
13631	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13632	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13633	{
13634	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13635	rcStrict = VINF_SUCCESS;
13636	}
13637	}
13638	else
13639	{
13640	/*
13641	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13642	*/
13643	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13644	AssertRCReturn(rc2, rc2);
13645
13646	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13647	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13648
13649	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13650	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13651
13652	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13653	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13654	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13655	}
13656	return rcStrict;
13657	}
13658
13659
13660	/**
13661	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13662	*/
13663	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13664	{
13665	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13666
13667	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13668	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13669	AssertRCReturn(rc, rc);
13670
13671	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13672	return VINF_EM_RAW_EMULATE_INSTR;
13673
13674	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13675	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13676	}
13677
13678
13679	/**
13680	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13681	*/
13682	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13683	{
13684	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13685
13686	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13687	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13688	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13689	AssertRCReturn(rc, rc);
13690
13691	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13692	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13693	{
13694	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13695	we must reset offsetting on VM-entry. See @bugref{6634}. */
13696	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13697	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13698	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13699	}
13700	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13701	{
13702	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13703	rcStrict = VINF_SUCCESS;
13704	}
13705	return rcStrict;
13706	}
13707
13708
13709	/**
13710	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13711	*/
13712	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13713	{
13714	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13715
13716	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13717	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13718	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13719	AssertRCReturn(rc, rc);
13720
13721	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13722	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13723	{
13724	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13725	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13726	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13727	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13728	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13729	}
13730	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13731	{
13732	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13733	rcStrict = VINF_SUCCESS;
13734	}
13735	return rcStrict;
13736	}
13737
13738
13739	/**
13740	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13741	*/
13742	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13743	{
13744	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13745
13746	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13747	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13748	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13749	AssertRCReturn(rc, rc);
13750
13751	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13752	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13753	if (RT_LIKELY(rc == VINF_SUCCESS))
13754	{
13755	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13756	Assert(pVmxTransient->cbInstr == 2);
13757	}
13758	else
13759	{
13760	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13761	rc = VERR_EM_INTERPRETER;
13762	}
13763	return rc;
13764	}
13765
13766
13767	/**
13768	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13769	*/
13770	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13771	{
13772	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13773
13774	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13775	if (EMAreHypercallInstructionsEnabled(pVCpu))
13776	{
13777	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13778	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13779	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13780	AssertRCReturn(rc, rc);
13781
13782	/* Perform the hypercall. */
13783	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13784	if (rcStrict == VINF_SUCCESS)
13785	{
13786	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13787	AssertRCReturn(rc, rc);
13788	}
13789	else
13790	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13791	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13792	\|\| RT_FAILURE(rcStrict));
13793
13794	/* If the hypercall changes anything other than guest's general-purpose registers,
13795	we would need to reload the guest changed bits here before VM-entry. */
13796	}
13797	else
13798	Log4Func(("Hypercalls not enabled\n"));
13799
13800	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13801	if (RT_FAILURE(rcStrict))
13802	{
13803	hmR0VmxSetPendingXcptUD(pVCpu);
13804	rcStrict = VINF_SUCCESS;
13805	}
13806
13807	return rcStrict;
13808	}
13809
13810
13811	/**
13812	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13813	*/
13814	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13815	{
13816	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13817	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13818
13819	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13820	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13821	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13822	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13823	AssertRCReturn(rc, rc);
13824
13825	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13826
13827	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13828	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13829	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13830	{
13831	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13832	rcStrict = VINF_SUCCESS;
13833	}
13834	else
13835	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
13836	VBOXSTRICTRC_VAL(rcStrict)));
13837	return rcStrict;
13838	}
13839
13840
13841	/**
13842	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13843	*/
13844	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13845	{
13846	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13847
13848	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13849	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
13850	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13851	AssertRCReturn(rc, rc);
13852
13853	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
13854	if (rcStrict == VINF_SUCCESS)
13855	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13856	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13857	{
13858	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13859	rcStrict = VINF_SUCCESS;
13860	}
13861
13862	return rcStrict;
13863	}
13864
13865
13866	/**
13867	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13868	*/
13869	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13870	{
13871	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13872
13873	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13874	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13875	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13876	AssertRCReturn(rc, rc);
13877
13878	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
13879	if (RT_SUCCESS(rcStrict))
13880	{
13881	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13882	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
13883	rcStrict = VINF_SUCCESS;
13884	}
13885
13886	return rcStrict;
13887	}
13888
13889
13890	/**
13891	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13892	* VM-exit.
13893	*/
13894	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13895	{
13896	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13897	return VINF_EM_RESET;
13898	}
13899
13900
13901	/**
13902	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13903	*/
13904	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13905	{
13906	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13907
13908	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13909	AssertRCReturn(rc, rc);
13910
13911	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13912	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13913	rc = VINF_SUCCESS;
13914	else
13915	rc = VINF_EM_HALT;
13916
13917	if (rc != VINF_SUCCESS)
13918	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13919	return rc;
13920	}
13921
13922
13923	/**
13924	* VM-exit handler for instructions that result in a \#UD exception delivered to
13925	* the guest.
13926	*/
13927	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13928	{
13929	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13930	hmR0VmxSetPendingXcptUD(pVCpu);
13931	return VINF_SUCCESS;
13932	}
13933
13934
13935	/**
13936	* VM-exit handler for expiry of the VMX-preemption timer.
13937	*/
13938	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13939	{
13940	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13941
13942	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13943	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13944
13945	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13946	PVM pVM = pVCpu->CTX_SUFF(pVM);
13947	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13949	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13950	}
13951
13952
13953	/**
13954	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13955	*/
13956	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13957	{
13958	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13959
13960	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13961	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13962	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13963	AssertRCReturn(rc, rc);
13964
13965	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13967	: HM_CHANGED_RAISED_XCPT_MASK);
13968
13969	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13970	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13971
13972	return rcStrict;
13973	}
13974
13975
13976	/**
13977	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13978	*/
13979	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13980	{
13981	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13982	/** @todo Use VM-exit instruction information. */
13983	return VERR_EM_INTERPRETER;
13984	}
13985
13986
13987	/**
13988	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13989	* Error VM-exit.
13990	*/
13991	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13992	{
13993	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13994	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13995	AssertRCReturn(rc, rc);
13996
13997	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13998	if (RT_FAILURE(rc))
13999	return rc;
14000
14001	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14002	NOREF(uInvalidReason);
14003
14004	#ifdef VBOX_STRICT
14005	uint32_t fIntrState;
14006	RTHCUINTREG uHCReg;
14007	uint64_t u64Val;
14008	uint32_t u32Val;
14009	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14010	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14011	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14012	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14013	AssertRCReturn(rc, rc);
14014
14015	Log4(("uInvalidReason %u\n", uInvalidReason));
14016	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14017	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14018	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14019	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14020
14021	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
14022	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
14023	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
14024	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
14025	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
14026	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14027	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
14028	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
14029	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
14030	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14031	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14032	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14033
14034	hmR0DumpRegs(pVCpu);
14035	#endif
14036
14037	return VERR_VMX_INVALID_GUEST_STATE;
14038	}
14039
14040	/**
14041	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14042	*/
14043	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14044	{
14045	/*
14046	* Cummulative notes of all recognized but unexpected VM-exits.
14047	* This does -not- cover those VM-exits like a page-fault occurring when say nested-paging
14048	* is used.
14049	*
14050	* VMX_EXIT_INIT_SIGNAL:
14051	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14052	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14053	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14054	*
14055	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14056	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14057	* See Intel spec. "23.8 Restrictions on VMX operation".
14058	*
14059	* VMX_EXIT_SIPI:
14060	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14061	* activity state is used. We don't make use of it as our guests don't have direct
14062	* access to the host local APIC.
14063	*
14064	* See Intel spec. 25.3 "Other Causes of VM-exits".
14065	*
14066	* VMX_EXIT_IO_SMI:
14067	* VMX_EXIT_SMI:
14068	* This can only happen if we support dual-monitor treatment of SMI, which can be
14069	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14070	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14071	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14072	*
14073	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14074	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14075	*
14076	* VMX_EXIT_ERR_MACHINE_CHECK:
14077	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14078	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14079	* #MC exception abort class exception is raised. We thus cannot assume a
14080	* reasonable chance of continuing any sort of execution and we bail.
14081	*
14082	* See Intel spec. 15.1 "Machine-check Architecture".
14083	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14084	*
14085	* VMX_EXIT_ERR_MSR_LOAD:
14086	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14087	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14088	* execution.
14089	*
14090	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14091	*
14092	* VMX_EXIT_PML_FULL:
14093	* VMX_EXIT_VIRTUALIZED_EOI:
14094	* VMX_EXIT_APIC_WRITE:
14095	* We do not currently support any of these features and thus they are all unexpected
14096	* VM-exits.
14097	*/
14098	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14099	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14100	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14101	}
14102
14103
14104	/**
14105	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
14106	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
14107	* Conditional VM-exit.
14108	*/
14109	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14110	{
14111	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14112
14113	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
14114	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
14115	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14116	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
14117	return VERR_EM_INTERPRETER;
14118	AssertMsgFailed(("Unexpected XDTR access\n"));
14119	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14120	}
14121
14122
14123	/**
14124	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
14125	*/
14126	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14127	{
14128	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14129
14130	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
14131	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14132	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
14133	return VERR_EM_INTERPRETER;
14134	AssertMsgFailed(("Unexpected RDRAND exit\n"));
14135	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14136	}
14137
14138
14139	/**
14140	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14141	*/
14142	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14143	{
14144	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14145
14146	/** @todo Optimize this: We currently drag in in the whole MSR state
14147	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14148	* MSRs required. That would require changes to IEM and possibly CPUM too.
14149	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14150	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14151	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14152	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14153	switch (idMsr)
14154	{
14155	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14156	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14157	}
14158
14159	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14160	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14161	AssertRCReturn(rc, rc);
14162
14163	Log4Func(("ecx=%#RX32\n", idMsr));
14164
14165	#ifdef VBOX_STRICT
14166	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14167	{
14168	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14169	&& idMsr != MSR_K6_EFER)
14170	{
14171	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14172	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14173	}
14174	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14175	{
14176	Assert(pVmcsInfo->pvMsrBitmap);
14177	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14178	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14179	{
14180	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14181	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14182	}
14183	}
14184	}
14185	#endif
14186
14187	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14189	if (rcStrict == VINF_SUCCESS)
14190	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14191	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14192	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14193	{
14194	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14195	rcStrict = VINF_SUCCESS;
14196	}
14197	else
14198	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14199
14200	return rcStrict;
14201	}
14202
14203
14204	/**
14205	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14206	*/
14207	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14208	{
14209	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14210
14211	/** @todo Optimize this: We currently drag in in the whole MSR state
14212	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14213	* MSRs required. That would require changes to IEM and possibly CPUM too.
14214	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14215	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14216	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14217
14218	/*
14219	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14220	* Although we don't need to fetch the base as it will be overwritten shortly, while
14221	* loading guest-state we would also load the entire segment register including limit
14222	* and attributes and thus we need to load them here.
14223	*/
14224	switch (idMsr)
14225	{
14226	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14227	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14228	}
14229
14230	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14231	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14232	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14233	AssertRCReturn(rc, rc);
14234
14235	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14236
14237	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14238	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14239
14240	if (rcStrict == VINF_SUCCESS)
14241	{
14242	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14243
14244	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14245	if ( idMsr == MSR_IA32_APICBASE
14246	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14247	&& idMsr <= MSR_IA32_X2APIC_END))
14248	{
14249	/*
14250	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14251	* When full APIC register virtualization is implemented we'll have to make
14252	* sure APIC state is saved from the VMCS before IEM changes it.
14253	*/
14254	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14255	}
14256	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14257	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14258	else if (idMsr == MSR_K6_EFER)
14259	{
14260	/*
14261	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14262	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14263	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14264	*/
14265	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14266	}
14267
14268	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14269	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14270	{
14271	switch (idMsr)
14272	{
14273	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14274	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14275	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14276	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14277	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14278	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14279	default:
14280	{
14281	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14282	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14283	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14284	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14285	break;
14286	}
14287	}
14288	}
14289	#ifdef VBOX_STRICT
14290	else
14291	{
14292	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14293	switch (idMsr)
14294	{
14295	case MSR_IA32_SYSENTER_CS:
14296	case MSR_IA32_SYSENTER_EIP:
14297	case MSR_IA32_SYSENTER_ESP:
14298	case MSR_K8_FS_BASE:
14299	case MSR_K8_GS_BASE:
14300	{
14301	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14302	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14303	}
14304
14305	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14306	default:
14307	{
14308	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14309	{
14310	/* EFER MSR writes are always intercepted. */
14311	if (idMsr != MSR_K6_EFER)
14312	{
14313	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14314	idMsr));
14315	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14316	}
14317	}
14318
14319	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14320	{
14321	Assert(pVmcsInfo->pvMsrBitmap);
14322	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14323	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14324	{
14325	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14326	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14327	}
14328	}
14329	break;
14330	}
14331	}
14332	}
14333	#endif /* VBOX_STRICT */
14334	}
14335	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14336	{
14337	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14338	rcStrict = VINF_SUCCESS;
14339	}
14340	else
14341	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14342
14343	return rcStrict;
14344	}
14345
14346
14347	/**
14348	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14349	*/
14350	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14351	{
14352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14353
14354	/** @todo The guest has likely hit a contended spinlock. We might want to
14355	* poke a schedule different guest VCPU. */
14356	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14357	if (RT_SUCCESS(rc))
14358	return VINF_EM_RAW_INTERRUPT;
14359
14360	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14361	return rc;
14362	}
14363
14364
14365	/**
14366	* VM-exit handler for when the TPR value is lowered below the specified
14367	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14368	*/
14369	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14370	{
14371	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14372	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14373
14374	/*
14375	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14376	* We'll re-evaluate pending interrupts and inject them before the next VM
14377	* entry so we can just continue execution here.
14378	*/
14379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14380	return VINF_SUCCESS;
14381	}
14382
14383
14384	/**
14385	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14386	* VM-exit.
14387	*
14388	* @retval VINF_SUCCESS when guest execution can continue.
14389	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14390	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
14391	* interpreter.
14392	*/
14393	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14394	{
14395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14396	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14397
14398	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14399	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14400	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14401	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14402	AssertRCReturn(rc, rc);
14403
14404	VBOXSTRICTRC rcStrict;
14405	PVM pVM = pVCpu->CTX_SUFF(pVM);
14406	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
14407	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14408	switch (uAccessType)
14409	{
14410	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14411	{
14412	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14413	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14414	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
14415	AssertMsg( rcStrict == VINF_SUCCESS
14416	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14417	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14418
14419	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14420	{
14421	case 0:
14422	{
14423	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14424	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14425	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14426	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14427
14428	/*
14429	* This is a kludge for handling switches back to real mode when we try to use
14430	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14431	* deal with special selector values, so we have to return to ring-3 and run
14432	* there till the selector values are V86 mode compatible.
14433	*
14434	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14435	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
14436	* at the end of this function.
14437	*/
14438	if ( rc == VINF_SUCCESS
14439	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
14440	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14441	&& (uOldCr0 & X86_CR0_PE)
14442	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
14443	{
14444	/** @todo check selectors rather than returning all the time. */
14445	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14446	rcStrict = VINF_EM_RESCHEDULE_REM;
14447	}
14448	break;
14449	}
14450
14451	case 2:
14452	{
14453	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14454	/* Nothing to do here, CR2 it's not part of the VMCS. */
14455	break;
14456	}
14457
14458	case 3:
14459	{
14460	Assert( !pVM->hm.s.fNestedPaging
14461	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14462	\|\| pVCpu->hm.s.fUsingDebugLoop);
14463	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14464	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14465	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14466	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14467	break;
14468	}
14469
14470	case 4:
14471	{
14472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14474	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14475	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14476	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14477	break;
14478	}
14479
14480	case 8:
14481	{
14482	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14483	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14484	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14485	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14486	break;
14487	}
14488	default:
14489	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
14490	break;
14491	}
14492	break;
14493	}
14494
14495	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14496	{
14497	Assert( !pVM->hm.s.fNestedPaging
14498	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14499	\|\| pVCpu->hm.s.fUsingDebugLoop
14500	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
14501	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14502	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
14503	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14504
14505	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
14506	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
14507	AssertMsg( rcStrict == VINF_SUCCESS
14508	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14509	#ifdef VBOX_WITH_STATISTICS
14510	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14511	{
14512	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14513	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14514	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14515	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14516	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14517	}
14518	#endif
14519	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14520	VBOXSTRICTRC_VAL(rcStrict)));
14521	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14522	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14523	else
14524	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14525	break;
14526	}
14527
14528	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
14529	{
14530	/*
14531	* CLTS (Clear Task-Switch Flag in CR0).
14532	*/
14533	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->cbInstr);
14534	break;
14535	}
14536
14537	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
14538	{
14539	/*
14540	* LMSW (Load Machine-Status Word into CR0).
14541	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
14542	*/
14543	RTGCPTR GCPtrEffDst;
14544	uint8_t const cbInstr = pVmxTransient->cbInstr;
14545	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
14546	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
14547	if (fMemOperand)
14548	{
14549	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14550	AssertRCReturn(rc, rc);
14551	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
14552	}
14553	else
14554	GCPtrEffDst = NIL_RTGCPTR;
14555
14556	rcStrict = hmR0VmxExitLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14557	break;
14558	}
14559
14560	default:
14561	{
14562	pVCpu->hm.s.u32HMError = uAccessType;
14563	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14564	VERR_VMX_UNEXPECTED_EXCEPTION);
14565	}
14566	}
14567
14568	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14569	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14570	if (rcStrict == VINF_IEM_RAISED_XCPT)
14571	{
14572	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14573	rcStrict = VINF_SUCCESS;
14574	}
14575
14576	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14577	NOREF(pVM);
14578	return rcStrict;
14579	}
14580
14581
14582	/**
14583	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14584	* VM-exit.
14585	*/
14586	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14587	{
14588	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14589	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14590
14591	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14592	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14593	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14594	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14595	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14596	\| CPUMCTX_EXTRN_EFER);
14597	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14598	AssertRCReturn(rc, rc);
14599
14600	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14601	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14602	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14603	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14604	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14605	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14606	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14607	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14608
14609	/*
14610	* Update exit history to see if this exit can be optimized.
14611	*/
14612	VBOXSTRICTRC rcStrict;
14613	PCEMEXITREC pExitRec = NULL;
14614	if ( !fGstStepping
14615	&& !fDbgStepping)
14616	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14617	!fIOString
14618	? !fIOWrite
14619	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14620	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14621	: !fIOWrite
14622	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14623	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14624	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14625	if (!pExitRec)
14626	{
14627	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14628	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14629
14630	uint32_t const cbValue = s_aIOSizes[uIOSize];
14631	uint32_t const cbInstr = pVmxTransient->cbInstr;
14632	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14633	PVM pVM = pVCpu->CTX_SUFF(pVM);
14634	if (fIOString)
14635	{
14636	/*
14637	* INS/OUTS - I/O String instruction.
14638	*
14639	* Use instruction-information if available, otherwise fall back on
14640	* interpreting the instruction.
14641	*/
14642	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14643	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14644	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14645	if (fInsOutsInfo)
14646	{
14647	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14648	AssertRCReturn(rc2, rc2);
14649	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14650	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14651	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14652	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14653	if (fIOWrite)
14654	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14655	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14656	else
14657	{
14658	/*
14659	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14660	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14661	* See Intel Instruction spec. for "INS".
14662	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14663	*/
14664	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14665	}
14666	}
14667	else
14668	rcStrict = IEMExecOne(pVCpu);
14669
14670	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14671	fUpdateRipAlready = true;
14672	}
14673	else
14674	{
14675	/*
14676	* IN/OUT - I/O instruction.
14677	*/
14678	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14679	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14680	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14681	if (fIOWrite)
14682	{
14683	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14685	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14686	&& !pCtx->eflags.Bits.u1TF)
14687	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14688	}
14689	else
14690	{
14691	uint32_t u32Result = 0;
14692	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14693	if (IOM_SUCCESS(rcStrict))
14694	{
14695	/* Save result of I/O IN instr. in AL/AX/EAX. */
14696	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14697	}
14698	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14699	&& !pCtx->eflags.Bits.u1TF)
14700	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14702	}
14703	}
14704
14705	if (IOM_SUCCESS(rcStrict))
14706	{
14707	if (!fUpdateRipAlready)
14708	{
14709	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14710	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14711	}
14712
14713	/*
14714	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14715	* while booting Fedora 17 64-bit guest.
14716	*
14717	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14718	*/
14719	if (fIOString)
14720	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14721
14722	/*
14723	* If any I/O breakpoints are armed, we need to check if one triggered
14724	* and take appropriate action.
14725	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14726	*/
14727	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14728	AssertRCReturn(rc, rc);
14729
14730	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14731	* execution engines about whether hyper BPs and such are pending. */
14732	uint32_t const uDr7 = pCtx->dr[7];
14733	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14734	&& X86_DR7_ANY_RW_IO(uDr7)
14735	&& (pCtx->cr4 & X86_CR4_DE))
14736	\|\| DBGFBpIsHwIoArmed(pVM)))
14737	{
14738	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14739
14740	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14741	VMMRZCallRing3Disable(pVCpu);
14742	HM_DISABLE_PREEMPT(pVCpu);
14743
14744	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14745
14746	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14747	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14748	{
14749	/* Raise #DB. */
14750	if (fIsGuestDbgActive)
14751	ASMSetDR6(pCtx->dr[6]);
14752	if (pCtx->dr[7] != uDr7)
14753	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14754
14755	hmR0VmxSetPendingXcptDB(pVCpu);
14756	}
14757	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14758	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14759	else if ( rcStrict2 != VINF_SUCCESS
14760	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14761	rcStrict = rcStrict2;
14762	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14763
14764	HM_RESTORE_PREEMPT();
14765	VMMRZCallRing3Enable(pVCpu);
14766	}
14767	}
14768
14769	#ifdef VBOX_STRICT
14770	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14771	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14772	Assert(!fIOWrite);
14773	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14774	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14775	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14776	Assert(fIOWrite);
14777	else
14778	{
14779	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14780	* statuses, that the VMM device and some others may return. See
14781	* IOM_SUCCESS() for guidance. */
14782	AssertMsg( RT_FAILURE(rcStrict)
14783	\|\| rcStrict == VINF_SUCCESS
14784	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14785	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14786	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14787	\|\| rcStrict == VINF_EM_RAW_TO_R3
14788	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14789	# endif
14790	}
14791	#endif
14792	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14793	}
14794	else
14795	{
14796	/*
14797	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14798	*/
14799	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14800	AssertRCReturn(rc2, rc2);
14801	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14802	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14803	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14804	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14805	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14806	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14807
14808	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14809	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14810
14811	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14812	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14813	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14814	}
14815	return rcStrict;
14816	}
14817
14818
14819	/**
14820	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14821	* VM-exit.
14822	*/
14823	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14824	{
14825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14826
14827	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14828	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14829	AssertRCReturn(rc, rc);
14830	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14831	{
14832	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14833	AssertRCReturn(rc, rc);
14834	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14835	{
14836	uint32_t uErrCode;
14837	RTGCUINTPTR GCPtrFaultAddress;
14838	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14839	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14840	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14841	if (fErrorCodeValid)
14842	{
14843	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14844	AssertRCReturn(rc, rc);
14845	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14846	}
14847	else
14848	uErrCode = 0;
14849
14850	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14851	&& uVector == X86_XCPT_PF)
14852	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14853	else
14854	GCPtrFaultAddress = 0;
14855
14856	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14857	AssertRCReturn(rc, rc);
14858
14859	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14860	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14861
14862	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14864	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14865	}
14866	}
14867
14868	/* Fall back to the interpreter to emulate the task-switch. */
14869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14870	return VERR_EM_INTERPRETER;
14871	}
14872
14873
14874	/**
14875	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14876	*/
14877	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14878	{
14879	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14880
14881	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14882	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14883	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14884	AssertRCReturn(rc, rc);
14885	return VINF_EM_DBG_STEPPED;
14886	}
14887
14888
14889	/**
14890	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14891	*/
14892	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14893	{
14894	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14896
14897	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14898	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14899	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14900	{
14901	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14902	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14903	{
14904	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14905	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14906	}
14907	}
14908	else
14909	{
14910	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14911	rcStrict1 = VINF_SUCCESS;
14912	return rcStrict1;
14913	}
14914
14915	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14916	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14917	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14918	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14919	AssertRCReturn(rc, rc);
14920
14921	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14922	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14923	VBOXSTRICTRC rcStrict2;
14924	switch (uAccessType)
14925	{
14926	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14927	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14928	{
14929	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14930	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14931	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14932
14933	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14934	GCPhys &= PAGE_BASE_GC_MASK;
14935	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14936	PVM pVM = pVCpu->CTX_SUFF(pVM);
14937	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14938	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14939
14940	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14941	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14942	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14943	CPUMCTX2CORE(pCtx), GCPhys);
14944	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14945	if ( rcStrict2 == VINF_SUCCESS
14946	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14947	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14948	{
14949	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14950	\| HM_CHANGED_GUEST_APIC_TPR);
14951	rcStrict2 = VINF_SUCCESS;
14952	}
14953	break;
14954	}
14955
14956	default:
14957	Log4Func(("uAccessType=%#x\n", uAccessType));
14958	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14959	break;
14960	}
14961
14962	if (rcStrict2 != VINF_SUCCESS)
14963	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14964	return rcStrict2;
14965	}
14966
14967
14968	/**
14969	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14970	* VM-exit.
14971	*/
14972	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14973	{
14974	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14975
14976	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14977	if (pVmxTransient->fWasGuestDebugStateActive)
14978	{
14979	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14980	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14981	}
14982
14983	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14984	if ( !pVCpu->hm.s.fSingleInstruction
14985	&& !pVmxTransient->fWasHyperDebugStateActive)
14986	{
14987	Assert(!DBGFIsStepping(pVCpu));
14988	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14989
14990	/* Don't intercept MOV DRx any more. */
14991	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14992	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14993	AssertRCReturn(rc, rc);
14994
14995	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14996	VMMRZCallRing3Disable(pVCpu);
14997	HM_DISABLE_PREEMPT(pVCpu);
14998
14999	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15000	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15001	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
15002
15003	HM_RESTORE_PREEMPT();
15004	VMMRZCallRing3Enable(pVCpu);
15005
15006	#ifdef VBOX_WITH_STATISTICS
15007	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15008	AssertRCReturn(rc, rc);
15009	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15010	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15011	else
15012	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15013	#endif
15014	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15015	return VINF_SUCCESS;
15016	}
15017
15018	/*
15019	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15020	* The EFER MSR is always up-to-date.
15021	* Update the segment registers and DR7 from the CPU.
15022	*/
15023	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15024	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15025	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15026	AssertRCReturn(rc, rc);
15027	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15028
15029	PVM pVM = pVCpu->CTX_SUFF(pVM);
15030	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15031	{
15032	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15033	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15034	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15035	if (RT_SUCCESS(rc))
15036	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15037	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15038	}
15039	else
15040	{
15041	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15042	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15043	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15044	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15045	}
15046
15047	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15048	if (RT_SUCCESS(rc))
15049	{
15050	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15051	AssertRCReturn(rc2, rc2);
15052	return VINF_SUCCESS;
15053	}
15054	return rc;
15055	}
15056
15057
15058	/**
15059	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15060	* Conditional VM-exit.
15061	*/
15062	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15063	{
15064	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15065	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15066
15067	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15068	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15069	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15070	{
15071	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
15072	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
15073	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15074	{
15075	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
15076	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15077	}
15078	}
15079	else
15080	{
15081	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15082	rcStrict1 = VINF_SUCCESS;
15083	return rcStrict1;
15084	}
15085
15086	/*
15087	* Get sufficent state and update the exit history entry.
15088	*/
15089	RTGCPHYS GCPhys;
15090	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15091	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15092	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15093	AssertRCReturn(rc, rc);
15094
15095	VBOXSTRICTRC rcStrict;
15096	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15097	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15098	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15099	if (!pExitRec)
15100	{
15101	/*
15102	* If we succeed, resume guest execution.
15103	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15104	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15105	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15106	* weird case. See @bugref{6043}.
15107	*/
15108	PVM pVM = pVCpu->CTX_SUFF(pVM);
15109	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15110	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15111	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15112	if ( rcStrict == VINF_SUCCESS
15113	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15114	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15115	{
15116	/* Successfully handled MMIO operation. */
15117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15118	\| HM_CHANGED_GUEST_APIC_TPR);
15119	rcStrict = VINF_SUCCESS;
15120	}
15121	}
15122	else
15123	{
15124	/*
15125	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15126	*/
15127	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15128	AssertRCReturn(rc2, rc2);
15129
15130	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15131	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15132
15133	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15134	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15135
15136	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15137	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15138	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15139	}
15140	return VBOXSTRICTRC_TODO(rcStrict);
15141	}
15142
15143
15144	/**
15145	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15146	* VM-exit.
15147	*/
15148	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15149	{
15150	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15151	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15152
15153	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15154	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15155	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15156	{
15157	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
15158	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15159	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
15160	}
15161	else
15162	{
15163	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15164	rcStrict1 = VINF_SUCCESS;
15165	return rcStrict1;
15166	}
15167
15168	RTGCPHYS GCPhys;
15169	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15170	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15171	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15172	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15173	AssertRCReturn(rc, rc);
15174
15175	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
15176	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
15177
15178	RTGCUINT uErrorCode = 0;
15179	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15180	uErrorCode \|= X86_TRAP_PF_ID;
15181	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15182	uErrorCode \|= X86_TRAP_PF_RW;
15183	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15184	uErrorCode \|= X86_TRAP_PF_P;
15185
15186	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15187
15188
15189	/* Handle the pagefault trap for the nested shadow table. */
15190	PVM pVM = pVCpu->CTX_SUFF(pVM);
15191	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15192
15193	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
15194	pCtx->cs.Sel, pCtx->rip));
15195
15196	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15197	TRPMResetTrap(pVCpu);
15198
15199	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15200	if ( rcStrict2 == VINF_SUCCESS
15201	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15202	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
15203	{
15204	/* Successfully synced our nested page tables. */
15205	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15206	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15207	return VINF_SUCCESS;
15208	}
15209
15210	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15211	return rcStrict2;
15212	}
15213
15214	/** @} */
15215
15216	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15217	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
15218	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15219
15220	/**
15221	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
15222	*/
15223	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15224	{
15225	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15226	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15227
15228	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15229	AssertRCReturn(rc, rc);
15230
15231	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15232	{
15233	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15234	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15235
15236	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15237	* provides VM-exit instruction length. If this causes problem later,
15238	* disassemble the instruction like it's done on AMD-V. */
15239	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15240	AssertRCReturn(rc2, rc2);
15241	return rc;
15242	}
15243
15244	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15245	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15246	return rc;
15247	}
15248
15249
15250	/**
15251	* VM-exit exception handler for \#BP (Breakpoint exception).
15252	*/
15253	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15254	{
15255	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15256	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15257
15258	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15259	AssertRCReturn(rc, rc);
15260
15261	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15262	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15263	if (rc == VINF_EM_RAW_GUEST_TRAP)
15264	{
15265	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15266	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15267	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15268	AssertRCReturn(rc, rc);
15269
15270	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15271	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15272	}
15273
15274	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15275	return rc;
15276	}
15277
15278
15279	/**
15280	* VM-exit exception handler for \#AC (alignment check exception).
15281	*/
15282	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15283	{
15284	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15285
15286	/*
15287	* Re-inject it. We'll detect any nesting before getting here.
15288	*/
15289	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15290	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15291	AssertRCReturn(rc, rc);
15292	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15293
15294	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15295	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15296	return VINF_SUCCESS;
15297	}
15298
15299
15300	/**
15301	* VM-exit exception handler for \#DB (Debug exception).
15302	*/
15303	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15304	{
15305	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15307
15308	/*
15309	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
15310	* for processing.
15311	*/
15312	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15313
15314	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15315	uint64_t const uDR6 = X86_DR6_INIT_VAL
15316	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15317	\| X86_DR6_BD \| X86_DR6_BS));
15318
15319	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15320	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15321	Log6Func(("rc=%Rrc\n", rc));
15322	if (rc == VINF_EM_RAW_GUEST_TRAP)
15323	{
15324	/*
15325	* The exception was for the guest. Update DR6, DR7.GD and
15326	* IA32_DEBUGCTL.LBR before forwarding it.
15327	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15328	*/
15329	VMMRZCallRing3Disable(pVCpu);
15330	HM_DISABLE_PREEMPT(pVCpu);
15331
15332	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15333	pCtx->dr[6] \|= uDR6;
15334	if (CPUMIsGuestDebugStateActive(pVCpu))
15335	ASMSetDR6(pCtx->dr[6]);
15336
15337	HM_RESTORE_PREEMPT();
15338	VMMRZCallRing3Enable(pVCpu);
15339
15340	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15341	AssertRCReturn(rc, rc);
15342
15343	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15344	pCtx->dr[7] &= ~X86_DR7_GD;
15345
15346	/* Paranoia. */
15347	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15348	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15349
15350	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15351	AssertRCReturn(rc, rc);
15352
15353	/*
15354	* Raise #DB in the guest.
15355	*
15356	* It is important to reflect exactly what the VM-exit gave us (preserving the
15357	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15358	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15359	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15360	*
15361	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15362	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15363	*/
15364	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15365	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15366	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15367	AssertRCReturn(rc, rc);
15368	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15369	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15370	return VINF_SUCCESS;
15371	}
15372
15373	/*
15374	* Not a guest trap, must be a hypervisor related debug event then.
15375	* Update DR6 in case someone is interested in it.
15376	*/
15377	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15378	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15379	CPUMSetHyperDR6(pVCpu, uDR6);
15380
15381	return rc;
15382	}
15383
15384
15385	/**
15386	* Hacks its way around the lovely mesa driver's backdoor accesses.
15387	*
15388	* @sa hmR0SvmHandleMesaDrvGp.
15389	*/
15390	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15391	{
15392	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15393	RT_NOREF(pCtx);
15394
15395	/* For now we'll just skip the instruction. */
15396	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15397	}
15398
15399
15400	/**
15401	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15402	* backdoor logging w/o checking what it is running inside.
15403	*
15404	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15405	* backdoor port and magic numbers loaded in registers.
15406	*
15407	* @returns true if it is, false if it isn't.
15408	* @sa hmR0SvmIsMesaDrvGp.
15409	*/
15410	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15411	{
15412	/* 0xed: IN eAX,dx */
15413	uint8_t abInstr[1];
15414	if (pVmxTransient->cbInstr != sizeof(abInstr))
15415	return false;
15416
15417	/* Check that it is #GP(0). */
15418	if (pVmxTransient->uExitIntErrorCode != 0)
15419	return false;
15420
15421	/* Check magic and port. */
15422	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15423	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15424	if (pCtx->rax != UINT32_C(0x564d5868))
15425	return false;
15426	if (pCtx->dx != UINT32_C(0x5658))
15427	return false;
15428
15429	/* Flat ring-3 CS. */
15430	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15431	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15432	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15433	if (pCtx->cs.Attr.n.u2Dpl != 3)
15434	return false;
15435	if (pCtx->cs.u64Base != 0)
15436	return false;
15437
15438	/* Check opcode. */
15439	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15440	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15441	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15442	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15443	if (RT_FAILURE(rc))
15444	return false;
15445	if (abInstr[0] != 0xed)
15446	return false;
15447
15448	return true;
15449	}
15450
15451
15452	/**
15453	* VM-exit exception handler for \#GP (General-protection exception).
15454	*
15455	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15456	*/
15457	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15458	{
15459	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15461
15462	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15463	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15464	if (pVmcsInfo->RealMode.fRealOnV86Active)
15465	{ /* likely */ }
15466	else
15467	{
15468	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15469	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15470	#endif
15471	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15472	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15473	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15474	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15475	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15476	AssertRCReturn(rc, rc);
15477	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15478	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15479
15480	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15481	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15482	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15483	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15484	else
15485	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15486	return rc;
15487	}
15488
15489	Assert(CPUMIsGuestInRealModeEx(pCtx));
15490	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15491
15492	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15493	AssertRCReturn(rc, rc);
15494
15495	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15496	if (rcStrict == VINF_SUCCESS)
15497	{
15498	if (!CPUMIsGuestInRealModeEx(pCtx))
15499	{
15500	/*
15501	* The guest is no longer in real-mode, check if we can continue executing the
15502	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15503	*/
15504	pVmcsInfo->RealMode.fRealOnV86Active = false;
15505	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15506	{
15507	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15508	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15509	}
15510	else
15511	{
15512	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15513	rcStrict = VINF_EM_RESCHEDULE;
15514	}
15515	}
15516	else
15517	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15518	}
15519	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15520	{
15521	rcStrict = VINF_SUCCESS;
15522	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15523	}
15524	return VBOXSTRICTRC_VAL(rcStrict);
15525	}
15526
15527
15528	/**
15529	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15530	* the exception reported in the VMX transient structure back into the VM.
15531	*
15532	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15533	* up-to-date.
15534	*/
15535	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15536	{
15537	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15538	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15539	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15540	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15541	("uVector=%#x u32XcptBitmap=%#X32\n",
15542	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15543	NOREF(pVmcsInfo);
15544	#endif
15545
15546	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15547	hmR0VmxCheckExitDueToEventDelivery(). */
15548	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15549	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15550	AssertRCReturn(rc, rc);
15551	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15552
15553	#ifdef DEBUG_ramshankar
15554	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15555	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15556	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15557	#endif
15558
15559	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15560	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15561	return VINF_SUCCESS;
15562	}
15563
15564
15565	/**
15566	* VM-exit exception handler for \#PF (Page-fault exception).
15567	*/
15568	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15569	{
15570	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15571	PVM pVM = pVCpu->CTX_SUFF(pVM);
15572	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15573	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15574	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15575	AssertRCReturn(rc, rc);
15576
15577	if (!pVM->hm.s.fNestedPaging)
15578	{ /* likely */ }
15579	else
15580	{
15581	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15582	Assert(pVCpu->hm.s.fUsingDebugLoop);
15583	#endif
15584	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15585	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15586	{
15587	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15588	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15589	}
15590	else
15591	{
15592	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15593	hmR0VmxSetPendingXcptDF(pVCpu);
15594	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15595	}
15596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15597	return rc;
15598	}
15599
15600	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15601	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15602	if (pVmxTransient->fVectoringPF)
15603	{
15604	Assert(pVCpu->hm.s.Event.fPending);
15605	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15606	}
15607
15608	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15609	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15610	AssertRCReturn(rc, rc);
15611
15612	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15613	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15614
15615	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15616	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15617
15618	Log4Func(("#PF: rc=%Rrc\n", rc));
15619	if (rc == VINF_SUCCESS)
15620	{
15621	/*
15622	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15623	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15624	*/
15625	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15626	TRPMResetTrap(pVCpu);
15627	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15628	return rc;
15629	}
15630
15631	if (rc == VINF_EM_RAW_GUEST_TRAP)
15632	{
15633	if (!pVmxTransient->fVectoringDoublePF)
15634	{
15635	/* It's a guest page fault and needs to be reflected to the guest. */
15636	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15637	TRPMResetTrap(pVCpu);
15638	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15639	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15640	uGstErrorCode, pVmxTransient->uExitQual);
15641	}
15642	else
15643	{
15644	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15645	TRPMResetTrap(pVCpu);
15646	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15647	hmR0VmxSetPendingXcptDF(pVCpu);
15648	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15649	}
15650
15651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15652	return VINF_SUCCESS;
15653	}
15654
15655	TRPMResetTrap(pVCpu);
15656	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15657	return rc;
15658	}
15659
15660
15661	/**
15662	* VM-exit exception handler for LMSW.
15663	*/
15664	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
15665	{
15666	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
15667	AssertMsg( rcStrict == VINF_SUCCESS
15668	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
15670	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15671
15672	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15673	if (rcStrict == VINF_IEM_RAISED_XCPT)
15674	{
15675	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15676	rcStrict = VINF_SUCCESS;
15677	}
15678	return rcStrict;
15679	}
15680
15681
15682	/**
15683	* VM-exit exception handler for CLTS.
15684	*/
15685	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, uint8_t cbInstr)
15686	{
15687	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
15688	AssertMsg( rcStrict == VINF_SUCCESS
15689	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15690	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
15691	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15692
15693	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15694	if (rcStrict == VINF_IEM_RAISED_XCPT)
15695	{
15696	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15697	rcStrict = VINF_SUCCESS;
15698	}
15699	return rcStrict;
15700	}
15701
15702	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15703	/** @name VMX instruction handlers.
15704	* @{
15705	*/
15706	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15707	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15708	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15709
15710	/**
15711	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15712	*/
15713	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15714	{
15715	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15716
15717	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15718	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15719	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15720	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15721	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15722	AssertRCReturn(rc, rc);
15723
15724	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15725
15726	VMXVEXITINFO ExitInfo;
15727	RT_ZERO(ExitInfo);
15728	ExitInfo.uReason = pVmxTransient->uExitReason;
15729	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15730	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15731	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15732	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15733
15734	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15735	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15736	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15737	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15738	{
15739	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15740	rcStrict = VINF_SUCCESS;
15741	}
15742	return rcStrict;
15743	}
15744
15745
15746	/**
15747	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15748	*/
15749	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15750	{
15751	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15752
15753	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15754	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15755	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15756	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15757	AssertRCReturn(rc, rc);
15758
15759	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15760
15761	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15762	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15763	{
15764	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15765	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15766	}
15767	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15768	return rcStrict;
15769	}
15770
15771
15772	/**
15773	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15774	*/
15775	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15776	{
15777	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15778
15779	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15780	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15781	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15782	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15783	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15784	AssertRCReturn(rc, rc);
15785
15786	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15787
15788	VMXVEXITINFO ExitInfo;
15789	RT_ZERO(ExitInfo);
15790	ExitInfo.uReason = pVmxTransient->uExitReason;
15791	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15792	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15793	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15794	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15795
15796	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15797	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15798	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15799	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15800	{
15801	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15802	rcStrict = VINF_SUCCESS;
15803	}
15804	return rcStrict;
15805	}
15806
15807
15808	/**
15809	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15810	*/
15811	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15812	{
15813	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15814
15815	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15816	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15817	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15818	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15819	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15820	AssertRCReturn(rc, rc);
15821
15822	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15823
15824	VMXVEXITINFO ExitInfo;
15825	RT_ZERO(ExitInfo);
15826	ExitInfo.uReason = pVmxTransient->uExitReason;
15827	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15828	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15829	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15830	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15831
15832	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15833	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15834	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15835	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15836	{
15837	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15838	rcStrict = VINF_SUCCESS;
15839	}
15840	return rcStrict;
15841	}
15842
15843
15844	/**
15845	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15846	*/
15847	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15848	{
15849	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15850
15851	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15852	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15853	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15854	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15855	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15856	AssertRCReturn(rc, rc);
15857
15858	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15859
15860	VMXVEXITINFO ExitInfo;
15861	RT_ZERO(ExitInfo);
15862	ExitInfo.uReason = pVmxTransient->uExitReason;
15863	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15864	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15865	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15866	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15867	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15868
15869	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15870	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15871	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15872	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15873	{
15874	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15875	rcStrict = VINF_SUCCESS;
15876	}
15877	return rcStrict;
15878	}
15879
15880
15881	/**
15882	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15883	*/
15884	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15885	{
15886	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15887
15888	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15889	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15890	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15891	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15892	AssertRCReturn(rc, rc);
15893
15894	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15895
15896	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15897	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15898	{
15899	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15901	}
15902	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15903	return rcStrict;
15904	}
15905
15906
15907	/**
15908	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15909	*/
15910	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15911	{
15912	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15913
15914	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15915	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15916	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15917	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15918	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15919	AssertRCReturn(rc, rc);
15920
15921	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15922
15923	VMXVEXITINFO ExitInfo;
15924	RT_ZERO(ExitInfo);
15925	ExitInfo.uReason = pVmxTransient->uExitReason;
15926	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15927	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15928	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15929	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15930	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15931
15932	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15933	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15934	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15935	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15936	{
15937	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15938	rcStrict = VINF_SUCCESS;
15939	}
15940	return rcStrict;
15941	}
15942
15943
15944	/**
15945	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15946	*/
15947	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15948	{
15949	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15950
15951	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15952	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15953	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15954	AssertRCReturn(rc, rc);
15955
15956	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15957
15958	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15959	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15960	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15961	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15962	{
15963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15964	rcStrict = VINF_SUCCESS;
15965	}
15966	return rcStrict;
15967	}
15968
15969
15970	/**
15971	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15972	*/
15973	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15974	{
15975	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15976
15977	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15978	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15979	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15980	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15981	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15982	AssertRCReturn(rc, rc);
15983
15984	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15985
15986	VMXVEXITINFO ExitInfo;
15987	RT_ZERO(ExitInfo);
15988	ExitInfo.uReason = pVmxTransient->uExitReason;
15989	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15990	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15991	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15992	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15993
15994	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15995	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15996	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15997	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15998	{
15999	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16000	rcStrict = VINF_SUCCESS;
16001	}
16002	return rcStrict;
16003	}
16004
16005
16006	/**
16007	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16008	*/
16009	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16010	{
16011	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16012
16013	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16014	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16015	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16016	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16017	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16018	AssertRCReturn(rc, rc);
16019
16020	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16021
16022	VMXVEXITINFO ExitInfo;
16023	RT_ZERO(ExitInfo);
16024	ExitInfo.uReason = pVmxTransient->uExitReason;
16025	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16026	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16027	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16028	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16029
16030	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16031	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16032	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16033	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16034	{
16035	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16036	rcStrict = VINF_SUCCESS;
16037	}
16038	return rcStrict;
16039	}
16040
16041	/** @} */
16042	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16043

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78777

Download in other formats: