HMVMXR0.cpp@ 78640

Last change on this file since 78640 was 78640, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 IEMExecVmxVmexitTaskSwitch takes care of figuring out which info. to use, caller can just pass everything.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 659.3 KB

Line
1	/* $Id: HMVMXR0.cpp 78640 2019-05-21 16:20:56Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
389	#endif
390	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
393	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
394	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
395	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
396	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
397	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
401	static FNVMXEXITHANDLER hmR0VmxExitMwait;
402	static FNVMXEXITHANDLER hmR0VmxExitMtf;
403	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
404	static FNVMXEXITHANDLER hmR0VmxExitPause;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
406	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
407	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
409	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
410	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
411	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
412	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
414	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
415	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
416	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
417	/** @} */
418
419	/** @name Helpers for hardware exceptions VM-exit handlers.
420	* @{
421	*/
422	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
429	/** @} */
430
431
432	/*********************************************************************************************************************************
433	* Global Variables *
434	*********************************************************************************************************************************/
435	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
436	static const uint32_t g_aVmcsCacheSegBase[] =
437	{
438	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
443	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
444	};
445	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
446	#endif
447	static const uint32_t g_aVmcsSegBase[] =
448	{
449	VMX_VMCS_GUEST_ES_BASE,
450	VMX_VMCS_GUEST_CS_BASE,
451	VMX_VMCS_GUEST_SS_BASE,
452	VMX_VMCS_GUEST_DS_BASE,
453	VMX_VMCS_GUEST_FS_BASE,
454	VMX_VMCS_GUEST_GS_BASE
455	};
456	static const uint32_t g_aVmcsSegSel[] =
457	{
458	VMX_VMCS16_GUEST_ES_SEL,
459	VMX_VMCS16_GUEST_CS_SEL,
460	VMX_VMCS16_GUEST_SS_SEL,
461	VMX_VMCS16_GUEST_DS_SEL,
462	VMX_VMCS16_GUEST_FS_SEL,
463	VMX_VMCS16_GUEST_GS_SEL
464	};
465	static const uint32_t g_aVmcsSegLimit[] =
466	{
467	VMX_VMCS32_GUEST_ES_LIMIT,
468	VMX_VMCS32_GUEST_CS_LIMIT,
469	VMX_VMCS32_GUEST_SS_LIMIT,
470	VMX_VMCS32_GUEST_DS_LIMIT,
471	VMX_VMCS32_GUEST_FS_LIMIT,
472	VMX_VMCS32_GUEST_GS_LIMIT
473	};
474	static const uint32_t g_aVmcsSegAttr[] =
475	{
476	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
481	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
482	};
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
486	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
487
488	#ifdef HMVMX_USE_FUNCTION_TABLE
489	/**
490	* VMX_EXIT dispatch table.
491	*/
492	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
493	{
494	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
495	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
496	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
497	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
498	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
499	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
500	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
501	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
502	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
503	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
504	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
505	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
506	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
507	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
508	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
509	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
510	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
511	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
512	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
513	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
514	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
515	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
516	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
517	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
518	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
519	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
520	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
521	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
522	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
523	#else
524	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
525	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
526	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
527	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
528	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
529	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
530	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
531	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
532	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
533	#endif
534	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
535	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
536	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
537	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
538	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
539	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
540	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
541	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
542	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
543	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
544	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
545	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
546	/* 40 UNDEFINED */ hmR0VmxExitPause,
547	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
548	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
549	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
550	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
551	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
552	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
554	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
555	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
556	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
557	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
558	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
559	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
560	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
561	#else
562	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
563	#endif
564	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
565	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
566	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
567	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
568	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
569	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
570	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
571	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
572	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
573	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
574	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
575	};
576	#endif /* HMVMX_USE_FUNCTION_TABLE */
577
578	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
579	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
580	{
581	/* 0 */ "(Not Used)",
582	/* 1 */ "VMCALL executed in VMX root operation.",
583	/* 2 */ "VMCLEAR with invalid physical address.",
584	/* 3 */ "VMCLEAR with VMXON pointer.",
585	/* 4 */ "VMLAUNCH with non-clear VMCS.",
586	/* 5 */ "VMRESUME with non-launched VMCS.",
587	/* 6 */ "VMRESUME after VMXOFF",
588	/* 7 */ "VM-entry with invalid control fields.",
589	/* 8 */ "VM-entry with invalid host state fields.",
590	/* 9 */ "VMPTRLD with invalid physical address.",
591	/* 10 */ "VMPTRLD with VMXON pointer.",
592	/* 11 */ "VMPTRLD with incorrect revision identifier.",
593	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
594	/* 13 */ "VMWRITE to read-only VMCS component.",
595	/* 14 */ "(Not Used)",
596	/* 15 */ "VMXON executed in VMX root operation.",
597	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
598	/* 17 */ "VM-entry with non-launched executing VMCS.",
599	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
600	/* 19 */ "VMCALL with non-clear VMCS.",
601	/* 20 */ "VMCALL with invalid VM-exit control fields.",
602	/* 21 */ "(Not Used)",
603	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
604	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
605	/* 24 */ "VMCALL with invalid SMM-monitor features.",
606	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
607	/* 26 */ "VM-entry with events blocked by MOV SS.",
608	/* 27 */ "(Not Used)",
609	/* 28 */ "Invalid operand to INVEPT/INVVPID."
610	};
611	#endif /* VBOX_STRICT */
612
613
614	/**
615	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
616	*
617	* Any bit set in this mask is owned by the host/hypervisor and would cause a
618	* VM-exit when modified by the guest.
619	*
620	* @returns The static CR0 guest/host mask.
621	* @param pVCpu The cross context virtual CPU structure.
622	*/
623	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
624	{
625	/*
626	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
627	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
628	*/
629	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
630	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
631	* and @bugref{6944}. */
632	PVM pVM = pVCpu->CTX_SUFF(pVM);
633	return ( X86_CR0_PE
634	\| X86_CR0_NE
635	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
636	\| X86_CR0_PG
637	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
638	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
639	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
640	}
641
642
643	/**
644	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
645	*
646	* Any bit set in this mask is owned by the host/hypervisor and would cause a
647	* VM-exit when modified by the guest.
648	*
649	* @returns The static CR4 guest/host mask.
650	* @param pVCpu The cross context virtual CPU structure.
651	*/
652	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
653	{
654	/*
655	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
656	* these bits are reserved on hardware that does not support them. Since the
657	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
658	* these bits and handle it depending on whether we expose them to the guest.
659	*/
660	PVM pVM = pVCpu->CTX_SUFF(pVM);
661	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
662	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
663	return ( X86_CR4_VMXE
664	\| X86_CR4_VME
665	\| X86_CR4_PAE
666	\| X86_CR4_PGE
667	\| X86_CR4_PSE
668	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
669	\| (fPcid ? X86_CR4_PCIDE : 0));
670	}
671
672
673	/**
674	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
675	* area.
676	*
677	* @returns @c true if it's different, @c false otherwise.
678	* @param pVmcsInfo The VMCS info. object.
679	*/
680	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
681	{
682	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
683	&& pVmcsInfo->pvGuestMsrStore);
684	}
685
686
687	/**
688	* Adds one or more exceptions to the exception bitmap and commits it to the current
689	* VMCS.
690	*
691	* @returns VBox status code.
692	* @param pVmxTransient The VMX-transient structure.
693	* @param uXcptMask The exception(s) to add.
694	*/
695	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
696	{
697	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
698	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
699	if ((uXcptBitmap & uXcptMask) != uXcptMask)
700	{
701	uXcptBitmap \|= uXcptMask;
702	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
703	AssertRCReturn(rc, rc);
704	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
705	}
706	return VINF_SUCCESS;
707	}
708
709
710	/**
711	* Adds an exception to the exception bitmap and commits it to the current VMCS.
712	*
713	* @returns VBox status code.
714	* @param pVmxTransient The VMX-transient structure.
715	* @param uXcpt The exception to add.
716	*/
717	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
718	{
719	Assert(uXcpt <= X86_XCPT_LAST);
720	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
721	}
722
723
724	/**
725	* Remove one or more exceptions from the exception bitmap and commits it to the
726	* current VMCS.
727	*
728	* This takes care of not removing the exception intercept if a nested-guest
729	* requires the exception to be intercepted.
730	*
731	* @returns VBox status code.
732	* @param pVCpu The cross context virtual CPU structure.
733	* @param pVmxTransient The VMX-transient structure.
734	* @param uXcptMask The exception(s) to remove.
735	*/
736	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
737	{
738	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
739	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
740	if (u32XcptBitmap & uXcptMask)
741	{
742	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
743	if (!pVmxTransient->fIsNestedGuest)
744	{ /* likely */ }
745	else
746	{
747	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
748	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
749	}
750	#endif
751	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
752	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
753	\| RT_BIT(X86_XCPT_DE)
754	\| RT_BIT(X86_XCPT_NM)
755	\| RT_BIT(X86_XCPT_TS)
756	\| RT_BIT(X86_XCPT_UD)
757	\| RT_BIT(X86_XCPT_NP)
758	\| RT_BIT(X86_XCPT_SS)
759	\| RT_BIT(X86_XCPT_GP)
760	\| RT_BIT(X86_XCPT_PF)
761	\| RT_BIT(X86_XCPT_MF));
762	#elif defined(HMVMX_ALWAYS_TRAP_PF)
763	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
764	#endif
765	if (uXcptMask)
766	{
767	/* Validate we are not removing any essential exception intercepts. */
768	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
769	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
770	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
771
772	/* Remove it from the exception bitmap. */
773	u32XcptBitmap &= ~uXcptMask;
774
775	/* Commit and update the cache if necessary. */
776	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
777	{
778	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
779	AssertRCReturn(rc, rc);
780	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
781	}
782	}
783	}
784	return VINF_SUCCESS;
785	}
786
787
788	/**
789	* Remove an exceptions from the exception bitmap and commits it to the current
790	* VMCS.
791	*
792	* @returns VBox status code.
793	* @param pVCpu The cross context virtual CPU structure.
794	* @param pVmxTransient The VMX-transient structure.
795	* @param uXcpt The exception to remove.
796	*/
797	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
798	{
799	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
800	}
801
802
803	/**
804	* Loads the VMCS specified by the VMCS info. object.
805	*
806	* @returns VBox status code.
807	* @param pVmcsInfo The VMCS info. object.
808	*/
809	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
810	{
811	Assert(pVmcsInfo);
812	Assert(pVmcsInfo->HCPhysVmcs);
813	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
814
815	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
816	{
817	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
818	if (RT_SUCCESS(rc))
819	{
820	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
821	return VINF_SUCCESS;
822	}
823	return rc;
824	}
825	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
826	}
827
828
829	/**
830	* Clears the VMCS specified by the VMCS info. object.
831	*
832	* @returns VBox status code.
833	* @param pVmcsInfo The VMCS info. object.
834	*/
835	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
836	{
837	Assert(pVmcsInfo);
838	Assert(pVmcsInfo->HCPhysVmcs);
839	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
840
841	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
842	if (RT_SUCCESS(rc))
843	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
844	return rc;
845	}
846
847
848	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
849	/**
850	* Switches the current VMCS to the one specified.
851	*
852	* @returns VBox status code.
853	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
854	* @param pVmcsInfoTo The VMCS info. object we are switching to.
855	*
856	* @remarks Called with interrupts disabled.
857	*/
858	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
859	{
860	Assert(pVmcsInfoFrom);
861	Assert(pVmcsInfoTo);
862
863	/*
864	* Clear the VMCS we are switching out if it has not already been cleared.
865	* This will sync any CPU internal data back to the VMCS.
866	*/
867	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
868	{
869	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
870	if (RT_SUCCESS(rc))
871	{ /* likely */ }
872	else
873	return rc;
874	}
875
876	/*
877	* Clear the VMCS we are switching to if it has not already been cleared.
878	* This will initialize the VMCS launch state to "clear" required for loading it.
879	*
880	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
881	*/
882	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
883	{
884	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
885	if (RT_SUCCESS(rc))
886	{ /* likely */ }
887	else
888	return rc;
889	}
890
891	/*
892	* Finally, load the VMCS we are switching to.
893	*/
894	return hmR0VmxLoadVmcs(pVmcsInfoTo);
895	}
896	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
897
898
899	/**
900	* Updates the VM's last error record.
901	*
902	* If there was a VMX instruction error, reads the error data from the VMCS and
903	* updates VCPU's last error record as well.
904	*
905	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
906	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
907	* VERR_VMX_INVALID_VMCS_FIELD.
908	* @param rc The error code.
909	*/
910	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
911	{
912	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
913	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
914	{
915	AssertPtrReturnVoid(pVCpu);
916	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
917	}
918	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
919	}
920
921
922	#ifdef VBOX_STRICT
923	/**
924	* Reads the VM-entry interruption-information field from the VMCS into the VMX
925	* transient structure.
926	*
927	* @returns VBox status code.
928	* @param pVmxTransient The VMX-transient structure.
929	*
930	* @remarks No-long-jump zone!!!
931	*/
932	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
933	{
934	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
935	AssertRCReturn(rc, rc);
936	return VINF_SUCCESS;
937	}
938
939
940	/**
941	* Reads the VM-entry exception error code field from the VMCS into
942	* the VMX transient structure.
943	*
944	* @returns VBox status code.
945	* @param pVmxTransient The VMX-transient structure.
946	*
947	* @remarks No-long-jump zone!!!
948	*/
949	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
950	{
951	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
952	AssertRCReturn(rc, rc);
953	return VINF_SUCCESS;
954	}
955
956
957	/**
958	* Reads the VM-entry exception error code field from the VMCS into
959	* the VMX transient structure.
960	*
961	* @returns VBox status code.
962	* @param pVmxTransient The VMX-transient structure.
963	*
964	* @remarks No-long-jump zone!!!
965	*/
966	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
967	{
968	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
969	AssertRCReturn(rc, rc);
970	return VINF_SUCCESS;
971	}
972	#endif /* VBOX_STRICT */
973
974
975	/**
976	* Reads the VM-exit interruption-information field from the VMCS into the VMX
977	* transient structure.
978	*
979	* @returns VBox status code.
980	* @param pVmxTransient The VMX-transient structure.
981	*/
982	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
983	{
984	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
985	{
986	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
987	AssertRCReturn(rc,rc);
988	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
989	}
990	return VINF_SUCCESS;
991	}
992
993
994	/**
995	* Reads the VM-exit interruption error code from the VMCS into the VMX
996	* transient structure.
997	*
998	* @returns VBox status code.
999	* @param pVmxTransient The VMX-transient structure.
1000	*/
1001	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1002	{
1003	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1004	{
1005	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1006	AssertRCReturn(rc, rc);
1007	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1008	}
1009	return VINF_SUCCESS;
1010	}
1011
1012
1013	/**
1014	* Reads the VM-exit instruction length field from the VMCS into the VMX
1015	* transient structure.
1016	*
1017	* @returns VBox status code.
1018	* @param pVmxTransient The VMX-transient structure.
1019	*/
1020	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1021	{
1022	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1023	{
1024	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1025	AssertRCReturn(rc, rc);
1026	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1027	}
1028	return VINF_SUCCESS;
1029	}
1030
1031
1032	/**
1033	* Reads the VM-exit instruction-information field from the VMCS into
1034	* the VMX transient structure.
1035	*
1036	* @returns VBox status code.
1037	* @param pVmxTransient The VMX-transient structure.
1038	*/
1039	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1040	{
1041	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1042	{
1043	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1044	AssertRCReturn(rc, rc);
1045	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1046	}
1047	return VINF_SUCCESS;
1048	}
1049
1050
1051	/**
1052	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1053	*
1054	* @returns VBox status code.
1055	* @param pVCpu The cross context virtual CPU structure of the
1056	* calling EMT. (Required for the VMCS cache case.)
1057	* @param pVmxTransient The VMX-transient structure.
1058	*/
1059	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1060	{
1061	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1062	{
1063	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1064	AssertRCReturn(rc, rc);
1065	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1066	}
1067	return VINF_SUCCESS;
1068	}
1069
1070
1071	/**
1072	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1073	*
1074	* @returns VBox status code.
1075	* @param pVCpu The cross context virtual CPU structure of the
1076	* calling EMT. (Required for the VMCS cache case.)
1077	* @param pVmxTransient The VMX-transient structure.
1078	*/
1079	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1080	{
1081	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1082	{
1083	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1084	AssertRCReturn(rc, rc);
1085	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1086	}
1087	return VINF_SUCCESS;
1088	}
1089
1090
1091	/**
1092	* Reads the IDT-vectoring information field from the VMCS into the VMX
1093	* transient structure.
1094	*
1095	* @returns VBox status code.
1096	* @param pVmxTransient The VMX-transient structure.
1097	*
1098	* @remarks No-long-jump zone!!!
1099	*/
1100	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1101	{
1102	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1103	{
1104	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1105	AssertRCReturn(rc, rc);
1106	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1107	}
1108	return VINF_SUCCESS;
1109	}
1110
1111
1112	/**
1113	* Reads the IDT-vectoring error code from the VMCS into the VMX
1114	* transient structure.
1115	*
1116	* @returns VBox status code.
1117	* @param pVmxTransient The VMX-transient structure.
1118	*/
1119	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1120	{
1121	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1122	{
1123	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1124	AssertRCReturn(rc, rc);
1125	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1126	}
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Enters VMX root mode operation on the current CPU.
1133	*
1134	* @returns VBox status code.
1135	* @param pVM The cross context VM structure. Can be
1136	* NULL, after a resume.
1137	* @param HCPhysCpuPage Physical address of the VMXON region.
1138	* @param pvCpuPage Pointer to the VMXON region.
1139	*/
1140	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1141	{
1142	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1143	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1144	Assert(pvCpuPage);
1145	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1146
1147	if (pVM)
1148	{
1149	/* Write the VMCS revision identifier to the VMXON region. */
1150	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1151	}
1152
1153	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1154	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1155
1156	/* Enable the VMX bit in CR4 if necessary. */
1157	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1158
1159	/* Enter VMX root mode. */
1160	int rc = VMXEnable(HCPhysCpuPage);
1161	if (RT_FAILURE(rc))
1162	{
1163	if (!(uOldCr4 & X86_CR4_VMXE))
1164	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1165
1166	if (pVM)
1167	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1168	}
1169
1170	/* Restore interrupts. */
1171	ASMSetFlags(fEFlags);
1172	return rc;
1173	}
1174
1175
1176	/**
1177	* Exits VMX root mode operation on the current CPU.
1178	*
1179	* @returns VBox status code.
1180	*/
1181	static int hmR0VmxLeaveRootMode(void)
1182	{
1183	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1184
1185	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1186	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1187
1188	/* If we're for some reason not in VMX root mode, then don't leave it. */
1189	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1190
1191	int rc;
1192	if (uHostCR4 & X86_CR4_VMXE)
1193	{
1194	/* Exit VMX root mode and clear the VMX bit in CR4. */
1195	VMXDisable();
1196	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1197	rc = VINF_SUCCESS;
1198	}
1199	else
1200	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1201
1202	/* Restore interrupts. */
1203	ASMSetFlags(fEFlags);
1204	return rc;
1205	}
1206
1207
1208	/**
1209	* Allocates and maps a physically contiguous page. The allocated page is
1210	* zero'd out (used by various VT-x structures).
1211	*
1212	* @returns IPRT status code.
1213	* @param pMemObj Pointer to the ring-0 memory object.
1214	* @param ppVirt Where to store the virtual address of the
1215	* allocation.
1216	* @param pHCPhys Where to store the physical address of the
1217	* allocation.
1218	*/
1219	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1220	{
1221	AssertPtr(pMemObj);
1222	AssertPtr(ppVirt);
1223	AssertPtr(pHCPhys);
1224	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1225	if (RT_FAILURE(rc))
1226	return rc;
1227	ppVirt = RTR0MemObjAddress(pMemObj);
1228	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1229	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1230	return VINF_SUCCESS;
1231	}
1232
1233
1234	/**
1235	* Frees and unmaps an allocated, physical page.
1236	*
1237	* @param pMemObj Pointer to the ring-0 memory object.
1238	* @param ppVirt Where to re-initialize the virtual address of
1239	* allocation as 0.
1240	* @param pHCPhys Where to re-initialize the physical address of the
1241	* allocation as 0.
1242	*/
1243	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1244	{
1245	AssertPtr(pMemObj);
1246	AssertPtr(ppVirt);
1247	AssertPtr(pHCPhys);
1248	/* NULL is valid, accepted and ignored by the free function below. */
1249	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1250	*pMemObj = NIL_RTR0MEMOBJ;
1251	*ppVirt = NULL;
1252	*pHCPhys = NIL_RTHCPHYS;
1253	}
1254
1255
1256	/**
1257	* Initializes a VMCS info. object.
1258	*
1259	* @param pVmcsInfo The VMCS info. object.
1260	*/
1261	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1262	{
1263	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1264
1265	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1266	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1267	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1268	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1269	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1270	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1272	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1273	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1274	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1275	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1276	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1277	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1278	}
1279
1280
1281	/**
1282	* Frees the VT-x structures for a VMCS info. object.
1283	*
1284	* @param pVM The cross context VM structure.
1285	* @param pVmcsInfo The VMCS info. object.
1286	*/
1287	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1288	{
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1290
1291	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1292	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1293
1294	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1295	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1296	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1297
1298	hmR0VmxInitVmcsInfo(pVmcsInfo);
1299	}
1300
1301
1302	/**
1303	* Allocates the VT-x structures for a VMCS info. object.
1304	*
1305	* @returns VBox status code.
1306	* @param pVCpu The cross context virtual CPU structure.
1307	* @param pVmcsInfo The VMCS info. object.
1308	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1309	*/
1310	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1311	{
1312	PVM pVM = pVCpu->CTX_SUFF(pVM);
1313
1314	/* Allocate the guest VM control structure (VMCS). */
1315	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1316	if (RT_SUCCESS(rc))
1317	{
1318	if (!fIsNstGstVmcs)
1319	{
1320	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1321	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1322	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1323	{
1324	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1325	NULL /* pR3Ptr /, NULL / pRCPtr */);
1326	}
1327	}
1328	else
1329	{
1330	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1331	Assert(!pVmcsInfo->pbVirtApic);
1332	}
1333
1334	if (RT_SUCCESS(rc))
1335	{
1336	/*
1337	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1338	* transparent accesses of specific MSRs.
1339	*
1340	* If the condition for enabling MSR bitmaps changes here, don't forget to
1341	* update HMIsMsrBitmapActive().
1342	*
1343	* We don't share MSR bitmaps between the guest and nested-guest as we then
1344	* don't need to care about carefully restoring the guest MSR bitmap.
1345	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1346	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1347	*/
1348	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1349	{
1350	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1351	if (RT_SUCCESS(rc))
1352	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1353	}
1354
1355	if (RT_SUCCESS(rc))
1356	{
1357	/*
1358	* Allocate the VM-entry MSR-load area for the guest MSRs.
1359	*
1360	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1361	* the guest and nested-guest.
1362	*/
1363	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1364	&pVmcsInfo->HCPhysGuestMsrLoad);
1365	if (RT_SUCCESS(rc))
1366	{
1367	/*
1368	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1369	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1370	*/
1371	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1372	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1373	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1374
1375	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1376	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1377	&pVmcsInfo->HCPhysHostMsrLoad);
1378	}
1379	}
1380	}
1381	}
1382
1383	return rc;
1384	}
1385
1386
1387	/**
1388	* Free all VT-x structures for the VM.
1389	*
1390	* @returns IPRT status code.
1391	* @param pVM The cross context VM structure.
1392	*/
1393	static void hmR0VmxStructsFree(PVM pVM)
1394	{
1395	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1396	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1397	#endif
1398	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1399
1400	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1401	{
1402	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1403	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1406	if (pVM->cpum.ro.GuestFeatures.fVmx)
1407	{
1408	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1409	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1410	}
1411	#endif
1412	}
1413	}
1414
1415
1416	/**
1417	* Allocate all VT-x structures for the VM.
1418	*
1419	* @returns IPRT status code.
1420	* @param pVM The cross context VM structure.
1421	*/
1422	static int hmR0VmxStructsAlloc(PVM pVM)
1423	{
1424	/*
1425	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1426	* The VMCS size cannot be more than 4096 bytes.
1427	*
1428	* See Intel spec. Appendix A.1 "Basic VMX Information".
1429	*/
1430	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1431	if (cbVmcs <= X86_PAGE_4K_SIZE)
1432	{ /* likely */ }
1433	else
1434	{
1435	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1436	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1437	}
1438
1439	/*
1440	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1441	*/
1442	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1443	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1445	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1446	#endif
1447
1448	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1449	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1450	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1451
1452	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1453	{
1454	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1455	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1456	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1457	}
1458
1459	/*
1460	* Allocate per-VM VT-x structures.
1461	*/
1462	int rc = VINF_SUCCESS;
1463	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1464	/* Allocate crash-dump magic scratch page. */
1465	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1466	if (RT_FAILURE(rc))
1467	{
1468	hmR0VmxStructsFree(pVM);
1469	return rc;
1470	}
1471	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1472	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1473	#endif
1474
1475	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1476	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1477	{
1478	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1479	&pVM->hm.s.vmx.HCPhysApicAccess);
1480	if (RT_FAILURE(rc))
1481	{
1482	hmR0VmxStructsFree(pVM);
1483	return rc;
1484	}
1485	}
1486
1487	/*
1488	* Initialize per-VCPU VT-x structures.
1489	*/
1490	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1491	{
1492	/* Allocate the guest VMCS structures. */
1493	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1494	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1495	if (RT_SUCCESS(rc))
1496	{
1497	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1498	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1499	if (pVM->cpum.ro.GuestFeatures.fVmx)
1500	{
1501	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1502	if (RT_SUCCESS(rc))
1503	{ /* likely */ }
1504	else
1505	break;
1506	}
1507	#endif
1508	}
1509	else
1510	break;
1511	}
1512
1513	if (RT_FAILURE(rc))
1514	{
1515	hmR0VmxStructsFree(pVM);
1516	return rc;
1517	}
1518
1519	return VINF_SUCCESS;
1520	}
1521
1522
1523	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1524	/**
1525	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1526	*
1527	* @returns @c true if the MSR is intercepted, @c false otherwise.
1528	* @param pvMsrBitmap The MSR bitmap.
1529	* @param offMsr The MSR byte offset.
1530	* @param iBit The bit offset from the byte offset.
1531	*/
1532	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1533	{
1534	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1535	Assert(pbMsrBitmap);
1536	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1537	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1538	}
1539	#endif
1540
1541
1542	/**
1543	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1544	*
1545	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1546	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1547	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1548	* the read/write access of this MSR.
1549	*
1550	* @param pVCpu The cross context virtual CPU structure.
1551	* @param pVmcsInfo The VMCS info. object.
1552	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1553	* @param idMsr The MSR value.
1554	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1555	* include both a read -and- a write permission!
1556	*
1557	* @sa CPUMGetVmxMsrPermission.
1558	* @remarks Can be called with interrupts disabled.
1559	*/
1560	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1561	{
1562	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1563	Assert(pbMsrBitmap);
1564	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1565
1566	/*
1567	* MSR-bitmap Layout:
1568	* Byte index MSR range Interpreted as
1569	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1570	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1571	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1572	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1573	*
1574	* A bit corresponding to an MSR within the above range causes a VM-exit
1575	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1576	* the MSR range, it always cause a VM-exit.
1577	*
1578	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1579	*/
1580	uint16_t const offBitmapRead = 0;
1581	uint16_t const offBitmapWrite = 0x800;
1582	uint16_t offMsr;
1583	int32_t iBit;
1584	if (idMsr <= UINT32_C(0x00001fff))
1585	{
1586	offMsr = 0;
1587	iBit = idMsr;
1588	}
1589	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1590	{
1591	offMsr = 0x400;
1592	iBit = idMsr - UINT32_C(0xc0000000);
1593	}
1594	else
1595	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1596
1597	/*
1598	* Set the MSR read permission.
1599	*/
1600	uint16_t const offMsrRead = offBitmapRead + offMsr;
1601	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1602	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1603	{
1604	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1605	bool const fClear = !fIsNstGstVmcs ? true
1606	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1607	#else
1608	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1609	bool const fClear = true;
1610	#endif
1611	if (fClear)
1612	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1613	}
1614	else
1615	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1616
1617	/*
1618	* Set the MSR write permission.
1619	*/
1620	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1621	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1622	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1623	{
1624	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1625	bool const fClear = !fIsNstGstVmcs ? true
1626	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1627	#else
1628	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1629	bool const fClear = true;
1630	#endif
1631	if (fClear)
1632	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1633	}
1634	else
1635	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1636	}
1637
1638
1639	/**
1640	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1641	* area.
1642	*
1643	* @returns VBox status code.
1644	* @param pVCpu The cross context virtual CPU structure.
1645	* @param pVmcsInfo The VMCS info. object.
1646	* @param cMsrs The number of MSRs.
1647	*/
1648	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1649	{
1650	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1651	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1652	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1653	{
1654	/* Commit the MSR counts to the VMCS and update the cache. */
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	{
1657	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1660	AssertRCReturn(rc, rc);
1661
1662	pVmcsInfo->cEntryMsrLoad = cMsrs;
1663	pVmcsInfo->cExitMsrStore = cMsrs;
1664	pVmcsInfo->cExitMsrLoad = cMsrs;
1665	}
1666	return VINF_SUCCESS;
1667	}
1668
1669	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1670	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1671	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1672	}
1673
1674
1675	/**
1676	* Adds a new (or updates the value of an existing) guest/host MSR
1677	* pair to be swapped during the world-switch as part of the
1678	* auto-load/store MSR area in the VMCS.
1679	*
1680	* @returns VBox status code.
1681	* @param pVCpu The cross context virtual CPU structure.
1682	* @param pVmxTransient The VMX-transient structure.
1683	* @param idMsr The MSR.
1684	* @param uGuestMsrValue Value of the guest MSR.
1685	* @param fSetReadWrite Whether to set the guest read/write access of this
1686	* MSR (thus not causing a VM-exit).
1687	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1688	* necessary.
1689	*/
1690	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1691	bool fSetReadWrite, bool fUpdateHostMsr)
1692	{
1693	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1694	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1695	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1696	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1697	uint32_t i;
1698
1699	/* Paranoia. */
1700	Assert(pGuestMsrLoad);
1701
1702	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1703
1704	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1705	for (i = 0; i < cMsrs; i++)
1706	{
1707	if (pGuestMsrLoad[i].u32Msr == idMsr)
1708	break;
1709	}
1710
1711	bool fAdded = false;
1712	if (i == cMsrs)
1713	{
1714	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1715	++cMsrs;
1716	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1717	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1718
1719	/* Set the guest to read/write this MSR without causing VM-exits. */
1720	if ( fSetReadWrite
1721	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1722	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1723
1724	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1725	fAdded = true;
1726	}
1727
1728	/* Update the MSR value for the newly added or already existing MSR. */
1729	pGuestMsrLoad[i].u32Msr = idMsr;
1730	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1731
1732	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1733	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1734	{
1735	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1736	pGuestMsrStore[i].u32Msr = idMsr;
1737	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1738	}
1739
1740	/* Update the corresponding slot in the host MSR area. */
1741	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1742	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1743	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1744	pHostMsr[i].u32Msr = idMsr;
1745
1746	/*
1747	* Only if the caller requests to update the host MSR value AND we've newly added the
1748	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1749	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1750	*
1751	* We do this for performance reasons since reading MSRs may be quite expensive.
1752	*/
1753	if (fAdded)
1754	{
1755	if (fUpdateHostMsr)
1756	{
1757	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1758	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1759	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1760	}
1761	else
1762	{
1763	/* Someone else can do the work. */
1764	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1765	}
1766	}
1767	return VINF_SUCCESS;
1768	}
1769
1770
1771	/**
1772	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1773	* auto-load/store MSR area in the VMCS.
1774	*
1775	* @returns VBox status code.
1776	* @param pVCpu The cross context virtual CPU structure.
1777	* @param pVmxTransient The VMX-transient structure.
1778	* @param idMsr The MSR.
1779	*/
1780	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1781	{
1782	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1783	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1784	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1785	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1786
1787	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1788
1789	for (uint32_t i = 0; i < cMsrs; i++)
1790	{
1791	/* Find the MSR. */
1792	if (pGuestMsrLoad[i].u32Msr == idMsr)
1793	{
1794	/*
1795	* If it's the last MSR, we only need to reduce the MSR count.
1796	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1797	*/
1798	if (i < cMsrs - 1)
1799	{
1800	/* Remove it from the VM-entry MSR-load area. */
1801	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1802	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1803
1804	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1805	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1806	{
1807	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1808	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1809	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1810	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1811	}
1812
1813	/* Remove it from the VM-exit MSR-load area. */
1814	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1815	Assert(pHostMsr[i].u32Msr == idMsr);
1816	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1817	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1818	}
1819
1820	/* Reduce the count to reflect the removed MSR and bail. */
1821	--cMsrs;
1822	break;
1823	}
1824	}
1825
1826	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1827	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1828	{
1829	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1830	AssertRCReturn(rc, rc);
1831
1832	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1833	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1834	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1835
1836	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1837	return VINF_SUCCESS;
1838	}
1839
1840	return VERR_NOT_FOUND;
1841	}
1842
1843
1844	/**
1845	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1846	*
1847	* @returns @c true if found, @c false otherwise.
1848	* @param pVmcsInfo The VMCS info. object.
1849	* @param idMsr The MSR to find.
1850	*/
1851	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1852	{
1853	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1854	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1855	Assert(pMsrs);
1856	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1857	for (uint32_t i = 0; i < cMsrs; i++)
1858	{
1859	if (pMsrs[i].u32Msr == idMsr)
1860	return true;
1861	}
1862	return false;
1863	}
1864
1865
1866	/**
1867	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1868	*
1869	* @param pVCpu The cross context virtual CPU structure.
1870	* @param pVmcsInfo The VMCS info. object.
1871	*
1872	* @remarks No-long-jump zone!!!
1873	*/
1874	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1875	{
1876	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1877
1878	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1879	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1880	Assert(pHostMsrLoad);
1881	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1882	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1883	for (uint32_t i = 0; i < cMsrs; i++)
1884	{
1885	/*
1886	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1887	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1888	*/
1889	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1890	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1891	else
1892	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1893	}
1894	}
1895
1896
1897	/**
1898	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1899	* perform lazy restoration of the host MSRs while leaving VT-x.
1900	*
1901	* @param pVCpu The cross context virtual CPU structure.
1902	*
1903	* @remarks No-long-jump zone!!!
1904	*/
1905	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1906	{
1907	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1908
1909	/*
1910	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1911	*/
1912	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1913	{
1914	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1915	#if HC_ARCH_BITS == 64
1916	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1917	{
1918	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1919	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1920	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1921	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1922	}
1923	#endif
1924	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1925	}
1926	}
1927
1928
1929	/**
1930	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1931	* lazily while leaving VT-x.
1932	*
1933	* @returns true if it does, false otherwise.
1934	* @param pVCpu The cross context virtual CPU structure.
1935	* @param idMsr The MSR to check.
1936	*/
1937	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1938	{
1939	NOREF(pVCpu);
1940	#if HC_ARCH_BITS == 64
1941	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1942	{
1943	switch (idMsr)
1944	{
1945	case MSR_K8_LSTAR:
1946	case MSR_K6_STAR:
1947	case MSR_K8_SF_MASK:
1948	case MSR_K8_KERNEL_GS_BASE:
1949	return true;
1950	}
1951	}
1952	#else
1953	RT_NOREF(pVCpu, idMsr);
1954	#endif
1955	return false;
1956	}
1957
1958
1959	/**
1960	* Loads a set of guests MSRs to allow read/passthru to the guest.
1961	*
1962	* The name of this function is slightly confusing. This function does NOT
1963	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1964	* common prefix for functions dealing with "lazy restoration" of the shared
1965	* MSRs.
1966	*
1967	* @param pVCpu The cross context virtual CPU structure.
1968	*
1969	* @remarks No-long-jump zone!!!
1970	*/
1971	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1972	{
1973	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1974	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1975
1976	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1977	#if HC_ARCH_BITS == 64
1978	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1979	{
1980	/*
1981	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1982	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1983	* we can skip a few MSR writes.
1984	*
1985	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1986	* guest MSR values in the guest-CPU context might be different to what's currently
1987	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1988	* CPU, see @bugref{8728}.
1989	*/
1990	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1991	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1992	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1993	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1994	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1995	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1996	{
1997	#ifdef VBOX_STRICT
1998	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1999	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2000	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2001	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2002	#endif
2003	}
2004	else
2005	{
2006	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2007	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2008	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2009	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2010	}
2011	}
2012	#endif
2013	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2014	}
2015
2016
2017	/**
2018	* Performs lazy restoration of the set of host MSRs if they were previously
2019	* loaded with guest MSR values.
2020	*
2021	* @param pVCpu The cross context virtual CPU structure.
2022	*
2023	* @remarks No-long-jump zone!!!
2024	* @remarks The guest MSRs should have been saved back into the guest-CPU
2025	* context by hmR0VmxImportGuestState()!!!
2026	*/
2027	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2028	{
2029	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2030	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2031
2032	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2033	{
2034	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2035	#if HC_ARCH_BITS == 64
2036	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2037	{
2038	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2039	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2040	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2041	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2042	}
2043	#endif
2044	}
2045	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2046	}
2047
2048
2049	/**
2050	* Verifies that our cached values of the VMCS fields are all consistent with
2051	* what's actually present in the VMCS.
2052	*
2053	* @returns VBox status code.
2054	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2055	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2056	* VMCS content. HMCPU error-field is
2057	* updated, see VMX_VCI_XXX.
2058	* @param pVCpu The cross context virtual CPU structure.
2059	* @param pVmcsInfo The VMCS info. object.
2060	*/
2061	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2062	{
2063	uint32_t u32Val;
2064	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2065	AssertRCReturn(rc, rc);
2066	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2067	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2068	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2069	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2070
2071	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2072	AssertRCReturn(rc, rc);
2073	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2074	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2075	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2076	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2077
2078	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2079	AssertRCReturn(rc, rc);
2080	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2081	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2082	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2083	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2084
2085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2086	AssertRCReturn(rc, rc);
2087	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2088	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2089	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2090	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2091
2092	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2093	{
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100	}
2101
2102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2105	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	uint64_t u64Val;
2110	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2111	AssertRCReturn(rc, rc);
2112	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2113	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2114	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2115	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2116
2117	return VINF_SUCCESS;
2118	}
2119
2120
2121	#ifdef VBOX_STRICT
2122	/**
2123	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2124	*
2125	* @param pVCpu The cross context virtual CPU structure.
2126	* @param pVmcsInfo The VMCS info. object.
2127	*/
2128	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2129	{
2130	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2131
2132	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2133	{
2134	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2135	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2136	uint64_t uVmcsEferMsrVmcs;
2137	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2138	AssertRC(rc);
2139
2140	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2141	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2142	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2143	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2144	}
2145	}
2146
2147
2148	/**
2149	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2150	* VMCS are correct.
2151	*
2152	* @param pVCpu The cross context virtual CPU structure.
2153	* @param pVmcsInfo The VMCS info. object.
2154	*/
2155	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2156	{
2157	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2158
2159	/* Read the various MSR-area counts from the VMCS. */
2160	uint32_t cEntryLoadMsrs;
2161	uint32_t cExitStoreMsrs;
2162	uint32_t cExitLoadMsrs;
2163	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2164	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2165	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2166
2167	/* Verify all the MSR counts are the same. */
2168	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2169	Assert(cExitStoreMsrs == cExitLoadMsrs);
2170	uint32_t const cMsrs = cExitLoadMsrs;
2171
2172	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2173	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2174
2175	/* Verify the MSR counts are within the allocated page size. */
2176	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2177
2178	/* Verify the relevant contents of the MSR areas match. */
2179	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2180	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2181	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2182	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2183	for (uint32_t i = 0; i < cMsrs; i++)
2184	{
2185	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2186	if (fSeparateExitMsrStorePage)
2187	{
2188	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2189	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2190	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2191	}
2192
2193	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2194	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2195	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2196
2197	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2198	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2199	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2200	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2201
2202	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2203	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2204	if (fIsEferMsr)
2205	{
2206	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2207	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2208	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2209	}
2210
2211	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2212	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2213	{
2214	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2215	if (fIsEferMsr)
2216	{
2217	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2218	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2219	}
2220	else
2221	{
2222	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2223	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2224	}
2225	}
2226
2227	/* Move to the next MSR. */
2228	pHostMsrLoad++;
2229	pGuestMsrLoad++;
2230	pGuestMsrStore++;
2231	}
2232	}
2233	#endif /* VBOX_STRICT */
2234
2235
2236	/**
2237	* Flushes the TLB using EPT.
2238	*
2239	* @returns VBox status code.
2240	* @param pVCpu The cross context virtual CPU structure of the calling
2241	* EMT. Can be NULL depending on @a enmTlbFlush.
2242	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2243	* enmTlbFlush.
2244	* @param enmTlbFlush Type of flush.
2245	*
2246	* @remarks Caller is responsible for making sure this function is called only
2247	* when NestedPaging is supported and providing @a enmTlbFlush that is
2248	* supported by the CPU.
2249	* @remarks Can be called with interrupts disabled.
2250	*/
2251	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2252	{
2253	uint64_t au64Descriptor[2];
2254	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2255	au64Descriptor[0] = 0;
2256	else
2257	{
2258	Assert(pVCpu);
2259	Assert(pVmcsInfo);
2260	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2261	}
2262	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2263
2264	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2265	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2266
2267	if ( RT_SUCCESS(rc)
2268	&& pVCpu)
2269	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2270	}
2271
2272
2273	/**
2274	* Flushes the TLB using VPID.
2275	*
2276	* @returns VBox status code.
2277	* @param pVCpu The cross context virtual CPU structure of the calling
2278	* EMT. Can be NULL depending on @a enmTlbFlush.
2279	* @param enmTlbFlush Type of flush.
2280	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2281	* on @a enmTlbFlush).
2282	*
2283	* @remarks Can be called with interrupts disabled.
2284	*/
2285	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2286	{
2287	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2288
2289	uint64_t au64Descriptor[2];
2290	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2291	{
2292	au64Descriptor[0] = 0;
2293	au64Descriptor[1] = 0;
2294	}
2295	else
2296	{
2297	AssertPtr(pVCpu);
2298	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2299	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2300	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2301	au64Descriptor[1] = GCPtr;
2302	}
2303
2304	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2305	AssertMsg(rc == VINF_SUCCESS,
2306	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2307
2308	if ( RT_SUCCESS(rc)
2309	&& pVCpu)
2310	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2311	NOREF(rc);
2312	}
2313
2314
2315	/**
2316	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2317	* otherwise there is nothing really to invalidate.
2318	*
2319	* @returns VBox status code.
2320	* @param pVCpu The cross context virtual CPU structure.
2321	* @param GCVirt Guest virtual address of the page to invalidate.
2322	*/
2323	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2324	{
2325	AssertPtr(pVCpu);
2326	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2327
2328	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2329	{
2330	/*
2331	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2332	* the EPT case. See @bugref{6043} and @bugref{6177}.
2333	*
2334	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2335	* as this function maybe called in a loop with individual addresses.
2336	*/
2337	PVM pVM = pVCpu->CTX_SUFF(pVM);
2338	if (pVM->hm.s.vmx.fVpid)
2339	{
2340	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2341
2342	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2343	/*
2344	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2345	* where executing INVVPID outside 64-bit mode does not flush translations of
2346	* 64-bit linear addresses, see @bugref{6208#c72}.
2347	*/
2348	if (RT_HI_U32(GCVirt))
2349	fVpidFlush = false;
2350	#endif
2351
2352	if (fVpidFlush)
2353	{
2354	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2355	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2356	}
2357	else
2358	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2359	}
2360	else if (pVM->hm.s.fNestedPaging)
2361	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2362	}
2363
2364	return VINF_SUCCESS;
2365	}
2366
2367
2368	/**
2369	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2370	* case where neither EPT nor VPID is supported by the CPU.
2371	*
2372	* @param pHostCpu The HM physical-CPU structure.
2373	* @param pVCpu The cross context virtual CPU structure.
2374	*
2375	* @remarks Called with interrupts disabled.
2376	*/
2377	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2378	{
2379	AssertPtr(pVCpu);
2380	AssertPtr(pHostCpu);
2381
2382	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2383
2384	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2385	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2386	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2387	pVCpu->hm.s.fForceTLBFlush = false;
2388	return;
2389	}
2390
2391
2392	/**
2393	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2394	*
2395	* @param pHostCpu The HM physical-CPU structure.
2396	* @param pVCpu The cross context virtual CPU structure.
2397	* @param pVmcsInfo The VMCS info. object.
2398	*
2399	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2400	* nomenclature. The reason is, to avoid confusion in compare statements
2401	* since the host-CPU copies are named "ASID".
2402	*
2403	* @remarks Called with interrupts disabled.
2404	*/
2405	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2406	{
2407	#ifdef VBOX_WITH_STATISTICS
2408	bool fTlbFlushed = false;
2409	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2410	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2411	if (!fTlbFlushed) \
2412	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2413	} while (0)
2414	#else
2415	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2416	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2417	#endif
2418
2419	AssertPtr(pVCpu);
2420	AssertPtr(pHostCpu);
2421	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2422
2423	PVM pVM = pVCpu->CTX_SUFF(pVM);
2424	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2425	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2426	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2427
2428	/*
2429	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2430	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2431	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2432	* cannot reuse the current ASID anymore.
2433	*/
2434	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2435	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2436	{
2437	++pHostCpu->uCurrentAsid;
2438	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2439	{
2440	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2441	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2442	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2443	}
2444
2445	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2446	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2447	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2448
2449	/*
2450	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2451	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2452	*/
2453	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2454	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2455	HMVMX_SET_TAGGED_TLB_FLUSHED();
2456	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2457	}
2458	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2459	{
2460	/*
2461	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2462	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2463	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2464	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2465	* mappings, see @bugref{6568}.
2466	*
2467	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2468	*/
2469	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2470	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2471	HMVMX_SET_TAGGED_TLB_FLUSHED();
2472	}
2473
2474	pVCpu->hm.s.fForceTLBFlush = false;
2475	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2476
2477	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2478	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2479	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2480	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2481	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2482	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2483	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2484	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2485	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2486
2487	/* Update VMCS with the VPID. */
2488	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2489	AssertRC(rc);
2490
2491	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2492	}
2493
2494
2495	/**
2496	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2497	*
2498	* @param pHostCpu The HM physical-CPU structure.
2499	* @param pVCpu The cross context virtual CPU structure.
2500	* @param pVmcsInfo The VMCS info. object.
2501	*
2502	* @remarks Called with interrupts disabled.
2503	*/
2504	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2505	{
2506	AssertPtr(pVCpu);
2507	AssertPtr(pHostCpu);
2508	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2509	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2510	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2511
2512	/*
2513	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2514	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2515	*/
2516	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2517	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2518	{
2519	pVCpu->hm.s.fForceTLBFlush = true;
2520	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2521	}
2522
2523	/* Check for explicit TLB flushes. */
2524	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2525	{
2526	pVCpu->hm.s.fForceTLBFlush = true;
2527	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2528	}
2529
2530	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2531	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2532
2533	if (pVCpu->hm.s.fForceTLBFlush)
2534	{
2535	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2536	pVCpu->hm.s.fForceTLBFlush = false;
2537	}
2538	}
2539
2540
2541	/**
2542	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2543	*
2544	* @param pHostCpu The HM physical-CPU structure.
2545	* @param pVCpu The cross context virtual CPU structure.
2546	*
2547	* @remarks Called with interrupts disabled.
2548	*/
2549	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2550	{
2551	AssertPtr(pVCpu);
2552	AssertPtr(pHostCpu);
2553	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2554	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2555	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2556
2557	/*
2558	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2559	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2560	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2561	* cannot reuse the current ASID anymore.
2562	*/
2563	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2564	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2565	{
2566	pVCpu->hm.s.fForceTLBFlush = true;
2567	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2568	}
2569
2570	/* Check for explicit TLB flushes. */
2571	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2572	{
2573	/*
2574	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2575	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2576	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2577	* include fExplicitFlush's too) - an obscure corner case.
2578	*/
2579	pVCpu->hm.s.fForceTLBFlush = true;
2580	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2581	}
2582
2583	PVM pVM = pVCpu->CTX_SUFF(pVM);
2584	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2585	if (pVCpu->hm.s.fForceTLBFlush)
2586	{
2587	++pHostCpu->uCurrentAsid;
2588	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2589	{
2590	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2591	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2592	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2593	}
2594
2595	pVCpu->hm.s.fForceTLBFlush = false;
2596	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2597	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2598	if (pHostCpu->fFlushAsidBeforeUse)
2599	{
2600	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2601	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2602	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2603	{
2604	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2605	pHostCpu->fFlushAsidBeforeUse = false;
2606	}
2607	else
2608	{
2609	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2610	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2611	}
2612	}
2613	}
2614
2615	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2616	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2617	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2618	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2619	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2620	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2621	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2622
2623	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2624	AssertRC(rc);
2625	}
2626
2627
2628	/**
2629	* Flushes the guest TLB entry based on CPU capabilities.
2630	*
2631	* @param pHostCpu The HM physical-CPU structure.
2632	* @param pVCpu The cross context virtual CPU structure.
2633	* @param pVmcsInfo The VMCS info. object.
2634	*
2635	* @remarks Called with interrupts disabled.
2636	*/
2637	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2638	{
2639	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2640	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2641	#endif
2642	PVM pVM = pVCpu->CTX_SUFF(pVM);
2643	switch (pVM->hm.s.vmx.enmTlbFlushType)
2644	{
2645	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2646	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2647	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2648	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2649	default:
2650	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2651	break;
2652	}
2653	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2654	}
2655
2656
2657	/**
2658	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2659	* TLB entries from the host TLB before VM-entry.
2660	*
2661	* @returns VBox status code.
2662	* @param pVM The cross context VM structure.
2663	*/
2664	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2665	{
2666	/*
2667	* Determine optimal flush type for nested paging.
2668	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2669	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2670	*/
2671	if (pVM->hm.s.fNestedPaging)
2672	{
2673	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2674	{
2675	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2676	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2677	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2678	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2679	else
2680	{
2681	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2682	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2683	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2684	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2685	}
2686
2687	/* Make sure the write-back cacheable memory type for EPT is supported. */
2688	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2689	{
2690	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2691	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2692	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2693	}
2694
2695	/* EPT requires a page-walk length of 4. */
2696	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2697	{
2698	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2699	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2700	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2701	}
2702	}
2703	else
2704	{
2705	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2706	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2707	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2708	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2709	}
2710	}
2711
2712	/*
2713	* Determine optimal flush type for VPID.
2714	*/
2715	if (pVM->hm.s.vmx.fVpid)
2716	{
2717	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2718	{
2719	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2720	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2721	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2722	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2723	else
2724	{
2725	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2726	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2727	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2728	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2729	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2730	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2731	pVM->hm.s.vmx.fVpid = false;
2732	}
2733	}
2734	else
2735	{
2736	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2737	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2738	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2739	pVM->hm.s.vmx.fVpid = false;
2740	}
2741	}
2742
2743	/*
2744	* Setup the handler for flushing tagged-TLBs.
2745	*/
2746	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2747	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2748	else if (pVM->hm.s.fNestedPaging)
2749	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2750	else if (pVM->hm.s.vmx.fVpid)
2751	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2752	else
2753	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2754	return VINF_SUCCESS;
2755	}
2756
2757
2758	/**
2759	* Sets up the virtual-APIC page address for the VMCS.
2760	*
2761	* @returns VBox status code.
2762	* @param pVCpu The cross context virtual CPU structure.
2763	* @param pVmcsInfo The VMCS info. object.
2764	*/
2765	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2766	{
2767	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2768	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2769	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2770	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2771	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2772	}
2773
2774
2775	/**
2776	* Sets up the MSR-bitmap address for the VMCS.
2777	*
2778	* @returns VBox status code.
2779	* @param pVCpu The cross context virtual CPU structure.
2780	* @param pVmcsInfo The VMCS info. object.
2781	*/
2782	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2783	{
2784	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2785	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2786	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2787	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2788	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2789	}
2790
2791
2792	/**
2793	* Sets up the APIC-access page address for the VMCS.
2794	*
2795	* @returns VBox status code.
2796	* @param pVCpu The cross context virtual CPU structure.
2797	*/
2798	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2799	{
2800	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2801	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2802	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2803	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2804	}
2805
2806
2807	/**
2808	* Sets up the VMCS link pointer for the VMCS.
2809	*
2810	* @returns VBox status code.
2811	* @param pVCpu The cross context virtual CPU structure.
2812	* @param pVmcsInfo The VMCS info. object.
2813	*/
2814	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2815	{
2816	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2817	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2818	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2819	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2820	}
2821
2822
2823	/**
2824	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2825	* in the VMCS.
2826	*
2827	* @returns VBox status code.
2828	* @param pVCpu The cross context virtual CPU structure.
2829	* @param pVmcsInfo The VMCS info. object.
2830	*/
2831	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2832	{
2833	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2834
2835	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2836	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2837	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2838
2839	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2840	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2841	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2842
2843	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2844	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2845	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2846
2847	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2848	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2849	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2850	AssertRCReturn(rc, rc);
2851	return VINF_SUCCESS;
2852	}
2853
2854
2855	/**
2856	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2857	*
2858	* @param pVCpu The cross context virtual CPU structure.
2859	* @param pVmcsInfo The VMCS info. object.
2860	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2861	*/
2862	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2863	{
2864	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2865
2866	/*
2867	* The guest can access the following MSRs (read, write) without causing
2868	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2869	*/
2870	PVM pVM = pVCpu->CTX_SUFF(pVM);
2871	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2872	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2873	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2874	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2875	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2876
2877	/*
2878	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2879	* associated with then. We never need to intercept access (writes need to be
2880	* executed without causing a VM-exit, reads will #GP fault anyway).
2881	*
2882	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2883	* read/write them. We swap the the guest/host MSR value using the
2884	* auto-load/store MSR area.
2885	*/
2886	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2887	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2888	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2889	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2890	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2891	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2892
2893	#if HC_ARCH_BITS == 64
2894	/*
2895	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2896	* required for 64-bit guests.
2897	*/
2898	if (pVM->hm.s.fAllow64BitGuests)
2899	{
2900	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2901	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2902	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2903	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2904	}
2905	#endif
2906
2907	/*
2908	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2909	*/
2910	#ifdef VBOX_STRICT
2911	Assert(pVmcsInfo->pvMsrBitmap);
2912	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2913	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2914	#endif
2915	}
2916
2917
2918	/**
2919	* Sets up pin-based VM-execution controls in the VMCS.
2920	*
2921	* @returns VBox status code.
2922	* @param pVCpu The cross context virtual CPU structure.
2923	* @param pVmcsInfo The VMCS info. object.
2924	*/
2925	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2926	{
2927	PVM pVM = pVCpu->CTX_SUFF(pVM);
2928	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2929	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2930
2931	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2932	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2933
2934	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2935	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2936
2937	/* Enable the VMX-preemption timer. */
2938	if (pVM->hm.s.vmx.fUsePreemptTimer)
2939	{
2940	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2941	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2942	}
2943
2944	#if 0
2945	/* Enable posted-interrupt processing. */
2946	if (pVM->hm.s.fPostedIntrs)
2947	{
2948	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2949	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2950	fVal \|= VMX_PIN_CTL_POSTED_INT;
2951	}
2952	#endif
2953
2954	if ((fVal & fZap) != fVal)
2955	{
2956	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2957	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2958	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2959	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2960	}
2961
2962	/* Commit it to the VMCS and update our cache. */
2963	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2964	AssertRCReturn(rc, rc);
2965	pVmcsInfo->u32PinCtls = fVal;
2966
2967	return VINF_SUCCESS;
2968	}
2969
2970
2971	/**
2972	* Sets up secondary processor-based VM-execution controls in the VMCS.
2973	*
2974	* @returns VBox status code.
2975	* @param pVCpu The cross context virtual CPU structure.
2976	* @param pVmcsInfo The VMCS info. object.
2977	*/
2978	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2979	{
2980	PVM pVM = pVCpu->CTX_SUFF(pVM);
2981	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2982	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2983
2984	/* WBINVD causes a VM-exit. */
2985	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2986	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2987
2988	/* Enable EPT (aka nested-paging). */
2989	if (pVM->hm.s.fNestedPaging)
2990	fVal \|= VMX_PROC_CTLS2_EPT;
2991
2992	/* Enable the INVPCID instruction if supported by the hardware and we expose
2993	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2994	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2995	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2996	fVal \|= VMX_PROC_CTLS2_INVPCID;
2997
2998	/* Enable VPID. */
2999	if (pVM->hm.s.vmx.fVpid)
3000	fVal \|= VMX_PROC_CTLS2_VPID;
3001
3002	/* Enable unrestricted guest execution. */
3003	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3004	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3005
3006	#if 0
3007	if (pVM->hm.s.fVirtApicRegs)
3008	{
3009	/* Enable APIC-register virtualization. */
3010	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3011	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3012
3013	/* Enable virtual-interrupt delivery. */
3014	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3015	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3016	}
3017	#endif
3018
3019	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3020	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3021	* done dynamically. */
3022	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3023	{
3024	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3025	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3026	AssertRCReturn(rc, rc);
3027	}
3028
3029	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3030	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3031	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3032	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3033	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3034
3035	/* Enable Pause-Loop exiting. */
3036	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3037	&& pVM->hm.s.vmx.cPleGapTicks
3038	&& pVM->hm.s.vmx.cPleWindowTicks)
3039	{
3040	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3041
3042	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3043	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3044	AssertRCReturn(rc, rc);
3045	}
3046
3047	if ((fVal & fZap) != fVal)
3048	{
3049	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3050	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3051	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3052	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3053	}
3054
3055	/* Commit it to the VMCS and update our cache. */
3056	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3057	AssertRCReturn(rc, rc);
3058	pVmcsInfo->u32ProcCtls2 = fVal;
3059
3060	return VINF_SUCCESS;
3061	}
3062
3063
3064	/**
3065	* Sets up processor-based VM-execution controls in the VMCS.
3066	*
3067	* @returns VBox status code.
3068	* @param pVCpu The cross context virtual CPU structure.
3069	* @param pVmcsInfo The VMCS info. object.
3070	*/
3071	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3072	{
3073	PVM pVM = pVCpu->CTX_SUFF(pVM);
3074
3075	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3076	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3077
3078	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3079	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3080	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3081	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3082	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3083	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3084	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3085
3086	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3087	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3088	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3089	{
3090	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3091	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3092	}
3093
3094	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3095	if (!pVM->hm.s.fNestedPaging)
3096	{
3097	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3098	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3099	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3100	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3101	}
3102
3103	/* Use TPR shadowing if supported by the CPU. */
3104	if ( PDMHasApic(pVM)
3105	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3106	{
3107	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3108	/* CR8 writes cause a VM-exit based on TPR threshold. */
3109	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3110	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3111	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3112	AssertRCReturn(rc, rc);
3113	}
3114	else
3115	{
3116	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3117	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3118	if (pVM->hm.s.fAllow64BitGuests)
3119	{
3120	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3121	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3122	}
3123	}
3124
3125	/* Use MSR-bitmaps if supported by the CPU. */
3126	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3127	{
3128	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3129	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3130	AssertRCReturn(rc, rc);
3131	}
3132
3133	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3134	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3135	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3136
3137	if ((fVal & fZap) != fVal)
3138	{
3139	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3140	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3141	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3143	}
3144
3145	/* Commit it to the VMCS and update our cache. */
3146	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3147	AssertRCReturn(rc, rc);
3148	pVmcsInfo->u32ProcCtls = fVal;
3149
3150	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3151	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3152	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3153
3154	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3155	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3156	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3157
3158	/* Sanity check, should not really happen. */
3159	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3160	{ /* likely */ }
3161	else
3162	{
3163	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3164	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3165	}
3166
3167	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3168	return VINF_SUCCESS;
3169	}
3170
3171
3172	/**
3173	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3174	* Processor-based VM-execution) control fields in the VMCS.
3175	*
3176	* @returns VBox status code.
3177	* @param pVCpu The cross context virtual CPU structure.
3178	* @param pVmcsInfo The VMCS info. object.
3179	*/
3180	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3181	{
3182	/* Set the auto-load/store MSR area addresses in the VMCS. */
3183	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3184	if (RT_SUCCESS(rc))
3185	{
3186	/* Set the VMCS link pointer in the VMCS. */
3187	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3188	if (RT_SUCCESS(rc))
3189	{
3190	/* Set the CR0/CR4 guest/host mask. */
3191	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3192	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3193	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3194	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3195	if (RT_SUCCESS(rc))
3196	{
3197	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3198	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3199	return VINF_SUCCESS;
3200	}
3201	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3202	}
3203	else
3204	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3205	}
3206	else
3207	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3208	return rc;
3209	}
3210
3211
3212	/**
3213	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3214	*
3215	* We shall setup those exception intercepts that don't change during the
3216	* lifetime of the VM here. The rest are done dynamically while loading the
3217	* guest state.
3218	*
3219	* @returns VBox status code.
3220	* @param pVCpu The cross context virtual CPU structure.
3221	* @param pVmcsInfo The VMCS info. object.
3222	*/
3223	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3224	{
3225	/*
3226	* The following exceptions are always intercepted:
3227	*
3228	* #AC - To prevent the guest from hanging the CPU.
3229	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3230	* recursive #DBs can cause a CPU hang.
3231	* #PF - To sync our shadow page tables when nested-paging is not used.
3232	*/
3233	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3234	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3235	\| RT_BIT(X86_XCPT_DB)
3236	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3237
3238	/* Commit it to the VMCS. */
3239	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3240	AssertRCReturn(rc, rc);
3241
3242	/* Update our cache of the exception bitmap. */
3243	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3244	return VINF_SUCCESS;
3245	}
3246
3247
3248	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3249	/**
3250	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3251	*
3252	* @returns VBox status code.
3253	* @param pVCpu The cross context virtual CPU structure.
3254	* @param pVmcsInfo The VMCS info. object.
3255	*/
3256	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3257	{
3258	PVM pVM = pVCpu->CTX_SUFF(pVM);
3259	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3260	if (RT_SUCCESS(rc))
3261	{
3262	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3263	if (RT_SUCCESS(rc))
3264	{
3265	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3266	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3267	if (RT_SUCCESS(rc))
3268	{
3269	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3270	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3271	if (RT_SUCCESS(rc))
3272	return VINF_SUCCESS;
3273
3274	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3275	}
3276	else
3277	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3278	}
3279	else
3280	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3281	}
3282	else
3283	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3284
3285	return rc;
3286	}
3287	#endif
3288
3289
3290	/**
3291	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3292	* VMX.
3293	*
3294	* @returns VBox status code.
3295	* @param pVCpu The cross context virtual CPU structure.
3296	* @param pVmcsInfo The VMCS info. object.
3297	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3298	*/
3299	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3300	{
3301	Assert(pVmcsInfo);
3302	Assert(pVmcsInfo->pvVmcs);
3303	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3304
3305	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3306	PVM pVM = pVCpu->CTX_SUFF(pVM);
3307	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3308	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3309
3310	LogFlowFunc(("\n"));
3311
3312	/*
3313	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3314	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3315	*/
3316	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3317	if (RT_SUCCESS(rc))
3318	{
3319	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3320	if (RT_SUCCESS(rc))
3321	{
3322	if (!fIsNstGstVmcs)
3323	{
3324	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3325	if (RT_SUCCESS(rc))
3326	{
3327	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3328	if (RT_SUCCESS(rc))
3329	{
3330	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3331	if (RT_SUCCESS(rc))
3332	{
3333	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3334	if (RT_SUCCESS(rc))
3335	{ /* likely */ }
3336	else
3337	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3338	}
3339	else
3340	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3341	}
3342	else
3343	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3344	}
3345	else
3346	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3347	}
3348	else
3349	{
3350	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3351	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3352	if (RT_SUCCESS(rc))
3353	{ /* likely */ }
3354	else
3355	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3356	#else
3357	AssertFailed();
3358	#endif
3359	}
3360	}
3361	else
3362	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3363	}
3364	else
3365	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3366
3367	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3368	if (RT_SUCCESS(rc))
3369	{
3370	rc = hmR0VmxClearVmcs(pVmcsInfo);
3371	if (RT_SUCCESS(rc))
3372	{ /* likely */ }
3373	else
3374	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3375	}
3376
3377	/*
3378	* Update the last-error record both for failures and success, so we
3379	* can propagate the status code back to ring-3 for diagnostics.
3380	*/
3381	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3382	NOREF(pszVmcs);
3383	return rc;
3384	}
3385
3386
3387	/**
3388	* Does global VT-x initialization (called during module initialization).
3389	*
3390	* @returns VBox status code.
3391	*/
3392	VMMR0DECL(int) VMXR0GlobalInit(void)
3393	{
3394	#ifdef HMVMX_USE_FUNCTION_TABLE
3395	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3396	# ifdef VBOX_STRICT
3397	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3398	Assert(g_apfnVMExitHandlers[i]);
3399	# endif
3400	#endif
3401	return VINF_SUCCESS;
3402	}
3403
3404
3405	/**
3406	* Does global VT-x termination (called during module termination).
3407	*/
3408	VMMR0DECL(void) VMXR0GlobalTerm()
3409	{
3410	/* Nothing to do currently. */
3411	}
3412
3413
3414	/**
3415	* Sets up and activates VT-x on the current CPU.
3416	*
3417	* @returns VBox status code.
3418	* @param pHostCpu The HM physical-CPU structure.
3419	* @param pVM The cross context VM structure. Can be
3420	* NULL after a host resume operation.
3421	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3422	* fEnabledByHost is @c true).
3423	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3424	* @a fEnabledByHost is @c true).
3425	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3426	* enable VT-x on the host.
3427	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3428	*/
3429	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3430	PCSUPHWVIRTMSRS pHwvirtMsrs)
3431	{
3432	Assert(pHostCpu);
3433	Assert(pHwvirtMsrs);
3434	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3435
3436	/* Enable VT-x if it's not already enabled by the host. */
3437	if (!fEnabledByHost)
3438	{
3439	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3440	if (RT_FAILURE(rc))
3441	return rc;
3442	}
3443
3444	/*
3445	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3446	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3447	* invalidated when flushing by VPID.
3448	*/
3449	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3450	{
3451	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3452	pHostCpu->fFlushAsidBeforeUse = false;
3453	}
3454	else
3455	pHostCpu->fFlushAsidBeforeUse = true;
3456
3457	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3458	++pHostCpu->cTlbFlushes;
3459
3460	return VINF_SUCCESS;
3461	}
3462
3463
3464	/**
3465	* Deactivates VT-x on the current CPU.
3466	*
3467	* @returns VBox status code.
3468	* @param pvCpuPage Pointer to the VMXON region.
3469	* @param HCPhysCpuPage Physical address of the VMXON region.
3470	*
3471	* @remarks This function should never be called when SUPR0EnableVTx() or
3472	* similar was used to enable VT-x on the host.
3473	*/
3474	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3475	{
3476	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3477
3478	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3479	return hmR0VmxLeaveRootMode();
3480	}
3481
3482
3483	/**
3484	* Does per-VM VT-x initialization.
3485	*
3486	* @returns VBox status code.
3487	* @param pVM The cross context VM structure.
3488	*/
3489	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3490	{
3491	LogFlowFunc(("pVM=%p\n", pVM));
3492
3493	int rc = hmR0VmxStructsAlloc(pVM);
3494	if (RT_FAILURE(rc))
3495	{
3496	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3497	return rc;
3498	}
3499
3500	return VINF_SUCCESS;
3501	}
3502
3503
3504	/**
3505	* Does per-VM VT-x termination.
3506	*
3507	* @returns VBox status code.
3508	* @param pVM The cross context VM structure.
3509	*/
3510	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3511	{
3512	LogFlowFunc(("pVM=%p\n", pVM));
3513
3514	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3515	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3516	{
3517	Assert(pVM->hm.s.vmx.pvScratch);
3518	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3519	}
3520	#endif
3521	hmR0VmxStructsFree(pVM);
3522	return VINF_SUCCESS;
3523	}
3524
3525
3526	/**
3527	* Sets up the VM for execution using hardware-assisted VMX.
3528	* This function is only called once per-VM during initialization.
3529	*
3530	* @returns VBox status code.
3531	* @param pVM The cross context VM structure.
3532	*/
3533	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3534	{
3535	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3536	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3537
3538	LogFlowFunc(("pVM=%p\n", pVM));
3539
3540	/*
3541	* At least verify if VMX is enabled, since we can't check if we're in
3542	* VMX root mode or not without causing a #GP.
3543	*/
3544	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3545	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3546	{ /* likely */ }
3547	else
3548	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3549
3550	/*
3551	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3552	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3553	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3554	*/
3555	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3556	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3557	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3558	{
3559	LogRelFunc(("Invalid real-on-v86 state.\n"));
3560	return VERR_INTERNAL_ERROR;
3561	}
3562
3563	/* Initialize these always, see hmR3InitFinalizeR0().*/
3564	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3565	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3566
3567	/* Setup the tagged-TLB flush handlers. */
3568	int rc = hmR0VmxSetupTaggedTlb(pVM);
3569	if (RT_FAILURE(rc))
3570	{
3571	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3572	return rc;
3573	}
3574
3575	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3576	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3577	#if HC_ARCH_BITS == 64
3578	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3579	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3580	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3581	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3582	#endif
3583
3584	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3585	{
3586	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3587	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3588
3589	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3590	if (RT_SUCCESS(rc))
3591	{
3592	#if HC_ARCH_BITS == 32
3593	hmR0VmxInitVmcsReadCache(pVCpu);
3594	#endif
3595	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3596	if (pVM->cpum.ro.GuestFeatures.fVmx)
3597	{
3598	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3599	if (RT_SUCCESS(rc))
3600	{ /* likely */ }
3601	else
3602	{
3603	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3604	return rc;
3605	}
3606	}
3607	#endif
3608	}
3609	else
3610	{
3611	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3612	return rc;
3613	}
3614	}
3615
3616	return VINF_SUCCESS;
3617	}
3618
3619
3620	#if HC_ARCH_BITS == 32
3621	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3622	/**
3623	* Check if guest state allows safe use of 32-bit switcher again.
3624	*
3625	* Segment bases and protected mode structures must be 32-bit addressable
3626	* because the 32-bit switcher will ignore high dword when writing these VMCS
3627	* fields. See @bugref{8432} for details.
3628	*
3629	* @returns true if safe, false if must continue to use the 64-bit switcher.
3630	* @param pCtx Pointer to the guest-CPU context.
3631	*
3632	* @remarks No-long-jump zone!!!
3633	*/
3634	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3635	{
3636	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3637	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3638	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3639	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3640	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3641	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3642	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3643	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3644	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3645	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3646
3647	/* All good, bases are 32-bit. */
3648	return true;
3649	}
3650	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3651
3652	# ifdef VBOX_STRICT
3653	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3654	{
3655	switch (idxField)
3656	{
3657	case VMX_VMCS_GUEST_RIP:
3658	case VMX_VMCS_GUEST_RSP:
3659	case VMX_VMCS_GUEST_SYSENTER_EIP:
3660	case VMX_VMCS_GUEST_SYSENTER_ESP:
3661	case VMX_VMCS_GUEST_GDTR_BASE:
3662	case VMX_VMCS_GUEST_IDTR_BASE:
3663	case VMX_VMCS_GUEST_CS_BASE:
3664	case VMX_VMCS_GUEST_DS_BASE:
3665	case VMX_VMCS_GUEST_ES_BASE:
3666	case VMX_VMCS_GUEST_FS_BASE:
3667	case VMX_VMCS_GUEST_GS_BASE:
3668	case VMX_VMCS_GUEST_SS_BASE:
3669	case VMX_VMCS_GUEST_LDTR_BASE:
3670	case VMX_VMCS_GUEST_TR_BASE:
3671	case VMX_VMCS_GUEST_CR3:
3672	return true;
3673	}
3674	return false;
3675	}
3676
3677	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3678	{
3679	switch (idxField)
3680	{
3681	/* Read-only fields. */
3682	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3683	return true;
3684	}
3685	/* Remaining readable fields should also be writable. */
3686	return hmR0VmxIsValidWriteField(idxField);
3687	}
3688	# endif /* VBOX_STRICT */
3689
3690
3691	/**
3692	* Executes the specified handler in 64-bit mode.
3693	*
3694	* @returns VBox status code (no informational status codes).
3695	* @param pVCpu The cross context virtual CPU structure.
3696	* @param enmOp The operation to perform.
3697	* @param cParams Number of parameters.
3698	* @param paParam Array of 32-bit parameters.
3699	*/
3700	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3701	{
3702	PVM pVM = pVCpu->CTX_SUFF(pVM);
3703	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3704	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3705	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3706	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3707
3708	#ifdef VBOX_STRICT
3709	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3710	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3711
3712	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3713	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3714	#endif
3715
3716	/* Disable interrupts. */
3717	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3718
3719	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3720	RTCPUID idHostCpu = RTMpCpuId();
3721	CPUMR0SetLApic(pVCpu, idHostCpu);
3722	#endif
3723
3724	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3725
3726	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3727	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3728
3729	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3730	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3731	hmR0VmxClearVmcs(pVmcsInfo);
3732
3733	/* Leave VMX root mode and disable VMX. */
3734	VMXDisable();
3735	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3736
3737	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3738	CPUMSetHyperEIP(pVCpu, enmOp);
3739	for (int i = (int)cParams - 1; i >= 0; i--)
3740	CPUMPushHyper(pVCpu, paParam[i]);
3741
3742	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3743
3744	/* Call the switcher. */
3745	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3746	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3747
3748	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3749	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3750
3751	/* Re-enter VMX root mode. */
3752	int rc2 = VMXEnable(HCPhysCpuPage);
3753	if (RT_FAILURE(rc2))
3754	{
3755	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3756	ASMSetFlags(fOldEFlags);
3757	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3758	return rc2;
3759	}
3760
3761	/* Restore the VMCS as the current VMCS. */
3762	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3763	AssertRC(rc2);
3764	Assert(!(ASMGetFlags() & X86_EFL_IF));
3765	ASMSetFlags(fOldEFlags);
3766	return rc;
3767	}
3768
3769
3770	/**
3771	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3772	* supporting 64-bit guests.
3773	*
3774	* @returns VBox status code.
3775	* @param fResume Whether to VMLAUNCH or VMRESUME.
3776	* @param pCtx Pointer to the guest-CPU context.
3777	* @param pCache Pointer to the VMCS batch cache.
3778	* @param pVM The cross context VM structure.
3779	* @param pVCpu The cross context virtual CPU structure.
3780	*/
3781	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3782	{
3783	NOREF(fResume);
3784
3785	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3786	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3787	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3788
3789	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3790	pCache->uPos = 1;
3791	pCache->interPD = PGMGetInterPaeCR3(pVM);
3792	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3793	#endif
3794
3795	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3796	pCache->TestIn.HCPhysCpuPage = 0;
3797	pCache->TestIn.HCPhysVmcs = 0;
3798	pCache->TestIn.pCache = 0;
3799	pCache->TestOut.HCPhysVmcs = 0;
3800	pCache->TestOut.pCache = 0;
3801	pCache->TestOut.pCtx = 0;
3802	pCache->TestOut.eflags = 0;
3803	#else
3804	NOREF(pCache);
3805	#endif
3806
3807	uint32_t aParam[10];
3808	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3809	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3810	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3811	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3812	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3813	aParam[5] = 0;
3814	aParam[6] = VM_RC_ADDR(pVM, pVM);
3815	aParam[7] = 0;
3816	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3817	aParam[9] = 0;
3818
3819	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3820	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3821	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3822	#endif
3823	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3824
3825	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3826	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3827	Assert(pCtx->dr[4] == 10);
3828	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3829	#endif
3830
3831	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3832	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3833	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3834	pVmcsInfo->HCPhysVmcs));
3835	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3836	pCache->TestOut.HCPhysVmcs));
3837	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3838	pCache->TestOut.pCache));
3839	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3840	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3841	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3842	pCache->TestOut.pCtx));
3843	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3844	#endif
3845	NOREF(pCtx);
3846	return rc;
3847	}
3848	#endif
3849
3850
3851	/**
3852	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3853	* the VMCS.
3854	*
3855	* @returns VBox status code.
3856	*/
3857	static int hmR0VmxExportHostControlRegs(void)
3858	{
3859	RTCCUINTREG uReg = ASMGetCR0();
3860	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3861	AssertRCReturn(rc, rc);
3862
3863	uReg = ASMGetCR3();
3864	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3865	AssertRCReturn(rc, rc);
3866
3867	uReg = ASMGetCR4();
3868	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3869	AssertRCReturn(rc, rc);
3870	return rc;
3871	}
3872
3873
3874	/**
3875	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3876	* the host-state area in the VMCS.
3877	*
3878	* @returns VBox status code.
3879	* @param pVCpu The cross context virtual CPU structure.
3880	*/
3881	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3882	{
3883	#if HC_ARCH_BITS == 64
3884	/**
3885	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3886	* requirements. See hmR0VmxExportHostSegmentRegs().
3887	*/
3888	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3889	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3890	{ \
3891	bool fValidSelector = true; \
3892	if ((selValue) & X86_SEL_LDT) \
3893	{ \
3894	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3895	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3896	} \
3897	if (fValidSelector) \
3898	{ \
3899	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3900	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3901	} \
3902	(selValue) = 0; \
3903	}
3904
3905	/*
3906	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3907	* will be messed up. We should -not- save the messed up state without restoring
3908	* the original host-state, see @bugref{7240}.
3909	*
3910	* This apparently can happen (most likely the FPU changes), deal with it rather than
3911	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3912	*/
3913	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3914	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3915	{
3916	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3917	pVCpu->idCpu));
3918	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3919	}
3920	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3921	#else
3922	RT_NOREF(pVCpu);
3923	#endif
3924
3925	/*
3926	* Host DS, ES, FS and GS segment registers.
3927	*/
3928	#if HC_ARCH_BITS == 64
3929	RTSEL uSelDS = ASMGetDS();
3930	RTSEL uSelES = ASMGetES();
3931	RTSEL uSelFS = ASMGetFS();
3932	RTSEL uSelGS = ASMGetGS();
3933	#else
3934	RTSEL uSelDS = 0;
3935	RTSEL uSelES = 0;
3936	RTSEL uSelFS = 0;
3937	RTSEL uSelGS = 0;
3938	#endif
3939
3940	/*
3941	* Host CS and SS segment registers.
3942	*/
3943	RTSEL uSelCS = ASMGetCS();
3944	RTSEL uSelSS = ASMGetSS();
3945
3946	/*
3947	* Host TR segment register.
3948	*/
3949	RTSEL uSelTR = ASMGetTR();
3950
3951	#if HC_ARCH_BITS == 64
3952	/*
3953	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3954	* gain VM-entry and restore them before we get preempted.
3955	*
3956	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3957	*/
3958	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3959	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3960	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3961	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3962	# undef VMXLOCAL_ADJUST_HOST_SEG
3963	#endif
3964
3965	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3966	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3967	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3968	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3969	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3970	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3971	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3972	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3973	Assert(uSelCS);
3974	Assert(uSelTR);
3975
3976	/* Write these host selector fields into the host-state area in the VMCS. */
3977	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3979	#if HC_ARCH_BITS == 64
3980	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3981	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3982	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3983	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3984	#else
3985	NOREF(uSelDS);
3986	NOREF(uSelES);
3987	NOREF(uSelFS);
3988	NOREF(uSelGS);
3989	#endif
3990	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3991	AssertRCReturn(rc, rc);
3992
3993	/*
3994	* Host GDTR and IDTR.
3995	*/
3996	RTGDTR Gdtr;
3997	RTIDTR Idtr;
3998	RT_ZERO(Gdtr);
3999	RT_ZERO(Idtr);
4000	ASMGetGDTR(&Gdtr);
4001	ASMGetIDTR(&Idtr);
4002	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4003	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4004	AssertRCReturn(rc, rc);
4005
4006	#if HC_ARCH_BITS == 64
4007	/*
4008	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4009	* them to the maximum limit (0xffff) on every VM-exit.
4010	*/
4011	if (Gdtr.cbGdt != 0xffff)
4012	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4013
4014	/*
4015	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4016	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4017	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4018	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4019	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4020	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4021	* at 0xffff on hosts where we are sure it won't cause trouble.
4022	*/
4023	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4024	if (Idtr.cbIdt < 0x0fff)
4025	# else
4026	if (Idtr.cbIdt != 0xffff)
4027	# endif
4028	{
4029	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4030	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4031	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4032	}
4033	#endif
4034
4035	/*
4036	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4037	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4038	* RPL should be too in most cases.
4039	*/
4040	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4041	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4042
4043	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4044	#if HC_ARCH_BITS == 64
4045	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4046
4047	/*
4048	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4049	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4050	* restoration if the host has something else. Task switching is not supported in 64-bit
4051	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4052	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4053	*
4054	* [1] See Intel spec. 3.5 "System Descriptor Types".
4055	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4056	*/
4057	PVM pVM = pVCpu->CTX_SUFF(pVM);
4058	Assert(pDesc->System.u4Type == 11);
4059	if ( pDesc->System.u16LimitLow != 0x67
4060	\|\| pDesc->System.u4LimitHigh)
4061	{
4062	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4063	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4064	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4065	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4066	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4067	}
4068
4069	/*
4070	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4071	*/
4072	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4073	{
4074	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4075	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4076	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4077	{
4078	/* The GDT is read-only but the writable GDT is available. */
4079	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4080	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4081	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4082	AssertRCReturn(rc, rc);
4083	}
4084	}
4085	#else
4086	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4087	#endif
4088	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4089	AssertRCReturn(rc, rc);
4090
4091	/*
4092	* Host FS base and GS base.
4093	*/
4094	#if HC_ARCH_BITS == 64
4095	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4096	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4097	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4098	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4099	AssertRCReturn(rc, rc);
4100
4101	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4102	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4103	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4104	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4105	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4106	#endif
4107	return VINF_SUCCESS;
4108	}
4109
4110
4111	/**
4112	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4113	* host-state area of the VMCS.
4114	*
4115	* These MSRs will be automatically restored on the host after every successful
4116	* VM-exit.
4117	*
4118	* @returns VBox status code.
4119	* @param pVCpu The cross context virtual CPU structure.
4120	*
4121	* @remarks No-long-jump zone!!!
4122	*/
4123	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4124	{
4125	AssertPtr(pVCpu);
4126
4127	/*
4128	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4129	* rather than swapping them on every VM-entry.
4130	*/
4131	hmR0VmxLazySaveHostMsrs(pVCpu);
4132
4133	/*
4134	* Host Sysenter MSRs.
4135	*/
4136	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4137	#if HC_ARCH_BITS == 32
4138	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4139	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4140	#else
4141	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4142	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4143	#endif
4144	AssertRCReturn(rc, rc);
4145
4146	/*
4147	* Host EFER MSR.
4148	*
4149	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4150	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4151	*/
4152	PVM pVM = pVCpu->CTX_SUFF(pVM);
4153	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4154	{
4155	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4156	AssertRCReturn(rc, rc);
4157	}
4158
4159	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4160	* hmR0VmxExportGuestEntryExitCtls(). */
4161
4162	return VINF_SUCCESS;
4163	}
4164
4165
4166	/**
4167	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4168	*
4169	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4170	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4171	*
4172	* @returns true if we need to load guest EFER, false otherwise.
4173	* @param pVCpu The cross context virtual CPU structure.
4174	*
4175	* @remarks Requires EFER, CR4.
4176	* @remarks No-long-jump zone!!!
4177	*/
4178	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4179	{
4180	#ifdef HMVMX_ALWAYS_SWAP_EFER
4181	RT_NOREF(pVCpu);
4182	return true;
4183	#else
4184	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4185	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4186	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4187	if (CPUMIsGuestInLongModeEx(pCtx))
4188	return false;
4189	#endif
4190
4191	PVM pVM = pVCpu->CTX_SUFF(pVM);
4192	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4193	uint64_t const u64GuestEfer = pCtx->msrEFER;
4194
4195	/*
4196	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4197	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4198	*/
4199	if ( CPUMIsGuestInLongModeEx(pCtx)
4200	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4201	return true;
4202
4203	/*
4204	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4205	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4206	*
4207	* See Intel spec. 4.5 "IA-32e Paging".
4208	* See Intel spec. 4.1.1 "Three Paging Modes".
4209	*
4210	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4211	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4212	*/
4213	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4214	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4215	if ( (pCtx->cr4 & X86_CR4_PAE)
4216	&& (pCtx->cr0 & X86_CR0_PG)
4217	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4218	{
4219	/* Assert that host is NX capable. */
4220	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4221	return true;
4222	}
4223
4224	return false;
4225	#endif
4226	}
4227
4228	/**
4229	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4230	* VMCS.
4231	*
4232	* This is typically required when the guest changes paging mode.
4233	*
4234	* @returns VBox status code.
4235	* @param pVCpu The cross context virtual CPU structure.
4236	* @param pVmxTransient The VMX-transient structure.
4237	*
4238	* @remarks Requires EFER.
4239	* @remarks No-long-jump zone!!!
4240	*/
4241	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4242	{
4243	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4244	{
4245	PVM pVM = pVCpu->CTX_SUFF(pVM);
4246	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4247
4248	/*
4249	* VM-entry controls.
4250	*/
4251	{
4252	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4253	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4254
4255	/*
4256	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4257	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4258	*
4259	* For nested-guests, this is a mandatory VM-entry control. It's also
4260	* required because we do not want to leak host bits to the nested-guest.
4261	*/
4262	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4263
4264	/*
4265	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4266	*
4267	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4268	* required to get the nested-guest working with hardware-assisted VMX execution.
4269	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4270	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4271	* here rather than while merging the guest VMCS controls.
4272	*/
4273	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4274	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4275	else
4276	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4277
4278	/*
4279	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4280	*
4281	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4282	* regardless of whether the nested-guest VMCS specifies it because we are free to
4283	* load whatever MSRs we require and we do not need to modify the guest visible copy
4284	* of the VM-entry MSR load area.
4285	*/
4286	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4287	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4288	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4289	else
4290	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4291
4292	/*
4293	* The following should -not- be set (since we're not in SMM mode):
4294	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4295	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4296	*/
4297
4298	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4299	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4300
4301	if ((fVal & fZap) == fVal)
4302	{ /* likely */ }
4303	else
4304	{
4305	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4306	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4307	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4308	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4309	}
4310
4311	/* Commit it to the VMCS. */
4312	if (pVmcsInfo->u32EntryCtls != fVal)
4313	{
4314	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4315	AssertRCReturn(rc, rc);
4316	pVmcsInfo->u32EntryCtls = fVal;
4317	}
4318	}
4319
4320	/*
4321	* VM-exit controls.
4322	*/
4323	{
4324	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4325	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4326
4327	/*
4328	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4329	* supported the 1-setting of this bit.
4330	*
4331	* For nested-guests, we set the "save debug controls" as the converse
4332	* "load debug controls" is mandatory for nested-guests anyway.
4333	*/
4334	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4335
4336	/*
4337	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4338	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4339	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4340	* hmR0VmxExportHostMsrs().
4341	*
4342	* For nested-guests, we always set this bit as we do not support 32-bit
4343	* hosts.
4344	*/
4345	#if HC_ARCH_BITS == 64
4346	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4347	#else
4348	Assert(!pVmxTransient->fIsNestedGuest);
4349	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4350	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4351	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4352	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4353	{
4354	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4355	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4356	}
4357	else
4358	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4359	#endif
4360
4361	/*
4362	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4363	*
4364	* For nested-guests, we should use the "save IA32_EFER" control if we also
4365	* used the "load IA32_EFER" control while exporting VM-entry controls.
4366	*/
4367	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4368	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4369	{
4370	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4371	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4372	}
4373
4374	/*
4375	* Enable saving of the VMX-preemption timer value on VM-exit.
4376	* For nested-guests, currently not exposed/used.
4377	*/
4378	if ( pVM->hm.s.vmx.fUsePreemptTimer
4379	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4380	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4381
4382	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4383	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4384
4385	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4386	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4387	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4388
4389	if ((fVal & fZap) == fVal)
4390	{ /* likely */ }
4391	else
4392	{
4393	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4394	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4395	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4396	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4397	}
4398
4399	/* Commit it to the VMCS. */
4400	if (pVmcsInfo->u32ExitCtls != fVal)
4401	{
4402	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4403	AssertRCReturn(rc, rc);
4404	pVmcsInfo->u32ExitCtls = fVal;
4405	}
4406	}
4407
4408	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4409	}
4410	return VINF_SUCCESS;
4411	}
4412
4413
4414	/**
4415	* Sets the TPR threshold in the VMCS.
4416	*
4417	* @returns VBox status code.
4418	* @param pVCpu The cross context virtual CPU structure.
4419	* @param pVmcsInfo The VMCS info. object.
4420	* @param u32TprThreshold The TPR threshold (task-priority class only).
4421	*/
4422	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4423	{
4424	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4425	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4426	RT_NOREF2(pVCpu, pVmcsInfo);
4427	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4428	}
4429
4430
4431	/**
4432	* Exports the guest APIC TPR state into the VMCS.
4433	*
4434	* @returns VBox status code.
4435	* @param pVCpu The cross context virtual CPU structure.
4436	* @param pVmxTransient The VMX-transient structure.
4437	*
4438	* @remarks No-long-jump zone!!!
4439	*/
4440	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4441	{
4442	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4443	{
4444	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4445
4446	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4447	if (!pVmxTransient->fIsNestedGuest)
4448	{
4449	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4450	&& APICIsEnabled(pVCpu))
4451	{
4452	/*
4453	* Setup TPR shadowing.
4454	*/
4455	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4456	{
4457	bool fPendingIntr = false;
4458	uint8_t u8Tpr = 0;
4459	uint8_t u8PendingIntr = 0;
4460	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4461	AssertRCReturn(rc, rc);
4462
4463	/*
4464	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4465	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4466	* priority of the pending interrupt so we can deliver the interrupt. If there
4467	* are no interrupts pending, set threshold to 0 to not cause any
4468	* TPR-below-threshold VM-exits.
4469	*/
4470	Assert(pVmcsInfo->pbVirtApic);
4471	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4472	uint32_t u32TprThreshold = 0;
4473	if (fPendingIntr)
4474	{
4475	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4476	(which is the Task-Priority Class). */
4477	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4478	const uint8_t u8TprPriority = u8Tpr >> 4;
4479	if (u8PendingPriority <= u8TprPriority)
4480	u32TprThreshold = u8PendingPriority;
4481	}
4482
4483	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4484	AssertRCReturn(rc, rc);
4485	}
4486	}
4487	}
4488	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4489	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4490	}
4491	return VINF_SUCCESS;
4492	}
4493
4494
4495	/**
4496	* Gets the guest interruptibility-state.
4497	*
4498	* @returns Guest's interruptibility-state.
4499	* @param pVCpu The cross context virtual CPU structure.
4500	* @param pVmcsInfo The VMCS info. object.
4501	*
4502	* @remarks No-long-jump zone!!!
4503	*/
4504	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4505	{
4506	/*
4507	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4508	*/
4509	uint32_t fIntrState = 0;
4510	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4511	{
4512	/* If inhibition is active, RIP and RFLAGS should've been updated
4513	(i.e. read previously from the VMCS or from ring-3). */
4514	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4515	#ifdef VBOX_STRICT
4516	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4517	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4518	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4519	#endif
4520	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4521	{
4522	if (pCtx->eflags.Bits.u1IF)
4523	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4524	else
4525	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4526	}
4527	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4528	{
4529	/*
4530	* We can clear the inhibit force flag as even if we go back to the recompiler
4531	* without executing guest code in VT-x, the flag's condition to be cleared is
4532	* met and thus the cleared state is correct.
4533	*/
4534	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4535	}
4536	}
4537
4538	/*
4539	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4540	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4541	* setting this would block host-NMIs and IRET will not clear the blocking.
4542	*
4543	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4544	*
4545	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4546	*/
4547	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4548	&& CPUMIsGuestNmiBlocking(pVCpu))
4549	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4550
4551	return fIntrState;
4552	}
4553
4554
4555	/**
4556	* Exports the exception intercepts required for guest execution in the VMCS.
4557	*
4558	* @returns VBox status code.
4559	* @param pVCpu The cross context virtual CPU structure.
4560	* @param pVmxTransient The VMX-transient structure.
4561	*
4562	* @remarks No-long-jump zone!!!
4563	*/
4564	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4565	{
4566	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4567	{
4568	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4569	if ( !pVmxTransient->fIsNestedGuest
4570	&& pVCpu->hm.s.fGIMTrapXcptUD)
4571	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4572	else
4573	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4574
4575	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4576	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4577	}
4578	return VINF_SUCCESS;
4579	}
4580
4581
4582	/**
4583	* Exports the guest's RIP into the guest-state area in the VMCS.
4584	*
4585	* @returns VBox status code.
4586	* @param pVCpu The cross context virtual CPU structure.
4587	*
4588	* @remarks No-long-jump zone!!!
4589	*/
4590	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4591	{
4592	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4593	{
4594	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4595
4596	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4597	AssertRCReturn(rc, rc);
4598
4599	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4600	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4601	}
4602	return VINF_SUCCESS;
4603	}
4604
4605
4606	/**
4607	* Exports the guest's RSP into the guest-state area in the VMCS.
4608	*
4609	* @returns VBox status code.
4610	* @param pVCpu The cross context virtual CPU structure.
4611	*
4612	* @remarks No-long-jump zone!!!
4613	*/
4614	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4615	{
4616	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4617	{
4618	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4619
4620	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4621	AssertRCReturn(rc, rc);
4622
4623	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4624	}
4625	return VINF_SUCCESS;
4626	}
4627
4628
4629	/**
4630	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4631	*
4632	* @returns VBox status code.
4633	* @param pVCpu The cross context virtual CPU structure.
4634	* @param pVmxTransient The VMX-transient structure.
4635	*
4636	* @remarks No-long-jump zone!!!
4637	*/
4638	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4639	{
4640	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4641	{
4642	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4643
4644	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4645	Let us assert it as such and use 32-bit VMWRITE. */
4646	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4647	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4648	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4649	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4650
4651	/*
4652	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4653	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4654	* can run the real-mode guest code under Virtual 8086 mode.
4655	*/
4656	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4657	if (pVmcsInfo->RealMode.fRealOnV86Active)
4658	{
4659	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4660	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4661	Assert(!pVmxTransient->fIsNestedGuest);
4662	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4663	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4664	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4665	}
4666
4667	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4668	AssertRCReturn(rc, rc);
4669
4670	/*
4671	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4672	*
4673	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4674	* through the hypervisor debugger using EFLAGS.TF.
4675	*/
4676	if ( !pVmxTransient->fIsNestedGuest
4677	&& !pVCpu->hm.s.fSingleInstruction
4678	&& fEFlags.Bits.u1TF)
4679	{
4680	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4681	* premature trips to ring-3 esp since IEM does not yet handle it. */
4682	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4683	AssertRCReturn(rc, rc);
4684	}
4685	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4686	* nested-guest VMCS. */
4687
4688	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4689	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4690	}
4691	return VINF_SUCCESS;
4692	}
4693
4694
4695	/**
4696	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4697	*
4698	* The guest FPU state is always pre-loaded hence we don't need to bother about
4699	* sharing FPU related CR0 bits between the guest and host.
4700	*
4701	* @returns VBox status code.
4702	* @param pVCpu The cross context virtual CPU structure.
4703	* @param pVmxTransient The VMX-transient structure.
4704	*
4705	* @remarks No-long-jump zone!!!
4706	*/
4707	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4708	{
4709	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4710	{
4711	PVM pVM = pVCpu->CTX_SUFF(pVM);
4712	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4713
4714	/*
4715	* Figure out fixed CR0 bits in VMX operation.
4716	*/
4717	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4718	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4719	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4720	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4721	else
4722	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4723
4724	if (!pVmxTransient->fIsNestedGuest)
4725	{
4726	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4727	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4728	uint64_t const u64ShadowCr0 = u64GuestCr0;
4729	Assert(!RT_HI_U32(u64GuestCr0));
4730
4731	/*
4732	* Setup VT-x's view of the guest CR0.
4733	*/
4734	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4735	if (pVM->hm.s.fNestedPaging)
4736	{
4737	if (CPUMIsGuestPagingEnabled(pVCpu))
4738	{
4739	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4740	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4741	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4742	}
4743	else
4744	{
4745	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4746	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4747	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4748	}
4749
4750	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4751	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4752	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4753	}
4754	else
4755	{
4756	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4757	u64GuestCr0 \|= X86_CR0_WP;
4758	}
4759
4760	/*
4761	* Guest FPU bits.
4762	*
4763	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4764	* using CR0.TS.
4765	*
4766	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4767	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4768	*/
4769	u64GuestCr0 \|= X86_CR0_NE;
4770
4771	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4772	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4773
4774	/*
4775	* Update exception intercepts.
4776	*/
4777	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4778	if (pVmcsInfo->RealMode.fRealOnV86Active)
4779	{
4780	Assert(PDMVmmDevHeapIsEnabled(pVM));
4781	Assert(pVM->hm.s.vmx.pRealModeTSS);
4782	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4783	}
4784	else
4785	{
4786	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4787	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4788	if (fInterceptMF)
4789	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4790	}
4791
4792	/* Additional intercepts for debugging, define these yourself explicitly. */
4793	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4794	uXcptBitmap \|= 0
4795	\| RT_BIT(X86_XCPT_BP)
4796	\| RT_BIT(X86_XCPT_DE)
4797	\| RT_BIT(X86_XCPT_NM)
4798	\| RT_BIT(X86_XCPT_TS)
4799	\| RT_BIT(X86_XCPT_UD)
4800	\| RT_BIT(X86_XCPT_NP)
4801	\| RT_BIT(X86_XCPT_SS)
4802	\| RT_BIT(X86_XCPT_GP)
4803	\| RT_BIT(X86_XCPT_PF)
4804	\| RT_BIT(X86_XCPT_MF)
4805	;
4806	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4807	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4808	#endif
4809	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4810	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4811	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4812
4813	/* Apply the fixed CR0 bits and enable caching. */
4814	u64GuestCr0 \|= fSetCr0;
4815	u64GuestCr0 &= fZapCr0;
4816	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4817
4818	/* Commit the CR0 and related fields to the guest VMCS. */
4819	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4820	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4821	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4822	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4823	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4824	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4825	AssertRCReturn(rc, rc);
4826
4827	/* Update our caches. */
4828	pVmcsInfo->u32ProcCtls = uProcCtls;
4829	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4830
4831	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4832	}
4833	else
4834	{
4835	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4836	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4837	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4838	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4839	Assert(!RT_HI_U32(u64GuestCr0));
4840	Assert(u64GuestCr0 & X86_CR0_NE);
4841
4842	/* Apply the fixed CR0 bits and enable caching. */
4843	u64GuestCr0 \|= fSetCr0;
4844	u64GuestCr0 &= fZapCr0;
4845	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4846
4847	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4848	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4849	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4850	AssertRCReturn(rc, rc);
4851
4852	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4853	}
4854
4855	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4856	}
4857
4858	return VINF_SUCCESS;
4859	}
4860
4861
4862	/**
4863	* Exports the guest control registers (CR3, CR4) into the guest-state area
4864	* in the VMCS.
4865	*
4866	* @returns VBox strict status code.
4867	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4868	* without unrestricted guest access and the VMMDev is not presently
4869	* mapped (e.g. EFI32).
4870	*
4871	* @param pVCpu The cross context virtual CPU structure.
4872	* @param pVmxTransient The VMX-transient structure.
4873	*
4874	* @remarks No-long-jump zone!!!
4875	*/
4876	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4877	{
4878	int rc = VINF_SUCCESS;
4879	PVM pVM = pVCpu->CTX_SUFF(pVM);
4880
4881	/*
4882	* Guest CR2.
4883	* It's always loaded in the assembler code. Nothing to do here.
4884	*/
4885
4886	/*
4887	* Guest CR3.
4888	*/
4889	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4890	{
4891	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4892
4893	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4894	if (pVM->hm.s.fNestedPaging)
4895	{
4896	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4897	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4898
4899	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4900	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4901	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4902	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4903
4904	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4905	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4906	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4907
4908	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4909	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4910	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4911	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4912	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4913	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4914	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4915
4916	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4917	AssertRCReturn(rc, rc);
4918
4919	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4920	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4921	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4922	{
4923	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4924	if (CPUMIsGuestInPAEModeEx(pCtx))
4925	{
4926	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4927	AssertRCReturn(rc, rc);
4928	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4929	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4930	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4931	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4932	AssertRCReturn(rc, rc);
4933	}
4934
4935	/*
4936	* The guest's view of its CR3 is unblemished with nested paging when the
4937	* guest is using paging or we have unrestricted guest execution to handle
4938	* the guest when it's not using paging.
4939	*/
4940	GCPhysGuestCR3 = pCtx->cr3;
4941	}
4942	else
4943	{
4944	/*
4945	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4946	* thinks it accesses physical memory directly, we use our identity-mapped
4947	* page table to map guest-linear to guest-physical addresses. EPT takes care
4948	* of translating it to host-physical addresses.
4949	*/
4950	RTGCPHYS GCPhys;
4951	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4952
4953	/* We obtain it here every time as the guest could have relocated this PCI region. */
4954	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4955	if (RT_SUCCESS(rc))
4956	{ /* likely */ }
4957	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4958	{
4959	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4960	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4961	}
4962	else
4963	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4964
4965	GCPhysGuestCR3 = GCPhys;
4966	}
4967
4968	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4969	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4970	AssertRCReturn(rc, rc);
4971	}
4972	else
4973	{
4974	/* Non-nested paging case, just use the hypervisor's CR3. */
4975	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4976
4977	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4978	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4979	AssertRCReturn(rc, rc);
4980	}
4981
4982	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4983	}
4984
4985	/*
4986	* Guest CR4.
4987	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4988	*/
4989	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4990	{
4991	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4992	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4993	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4994
4995	/*
4996	* Figure out fixed CR4 bits in VMX operation.
4997	*/
4998	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4999	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5000
5001	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5002	uint64_t u64GuestCr4 = pCtx->cr4;
5003	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
5004	Assert(!RT_HI_U32(u64GuestCr4));
5005
5006	/*
5007	* Setup VT-x's view of the guest CR4.
5008	*
5009	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5010	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5011	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5012	*
5013	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5014	*/
5015	if (pVmcsInfo->RealMode.fRealOnV86Active)
5016	{
5017	Assert(pVM->hm.s.vmx.pRealModeTSS);
5018	Assert(PDMVmmDevHeapIsEnabled(pVM));
5019	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5020	}
5021
5022	if (pVM->hm.s.fNestedPaging)
5023	{
5024	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5025	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5026	{
5027	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5028	u64GuestCr4 \|= X86_CR4_PSE;
5029	/* Our identity mapping is a 32-bit page directory. */
5030	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5031	}
5032	/* else use guest CR4.*/
5033	}
5034	else
5035	{
5036	Assert(!pVmxTransient->fIsNestedGuest);
5037
5038	/*
5039	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5040	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5041	*/
5042	switch (pVCpu->hm.s.enmShadowMode)
5043	{
5044	case PGMMODE_REAL: /* Real-mode. */
5045	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5046	case PGMMODE_32_BIT: /* 32-bit paging. */
5047	{
5048	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5049	break;
5050	}
5051
5052	case PGMMODE_PAE: /* PAE paging. */
5053	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5054	{
5055	u64GuestCr4 \|= X86_CR4_PAE;
5056	break;
5057	}
5058
5059	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5060	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5061	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5062	break;
5063	#endif
5064	default:
5065	AssertFailed();
5066	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5067	}
5068	}
5069
5070	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5071	u64GuestCr4 \|= fSetCr4;
5072	u64GuestCr4 &= fZapCr4;
5073
5074	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5075	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4); /** @todo Fix to 64-bit when we drop 32-bit. */
5076	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5077	AssertRCReturn(rc, rc);
5078
5079	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5080	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5081
5082	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5083
5084	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5085	}
5086	return rc;
5087	}
5088
5089
5090	/**
5091	* Exports the guest debug registers into the guest-state area in the VMCS.
5092	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5093	*
5094	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5095	*
5096	* @returns VBox status code.
5097	* @param pVCpu The cross context virtual CPU structure.
5098	* @param pVmxTransient The VMX-transient structure.
5099	*
5100	* @remarks No-long-jump zone!!!
5101	*/
5102	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5103	{
5104	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5105
5106	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5107	* stepping. */
5108	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5109	if (pVmxTransient->fIsNestedGuest)
5110	{
5111	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5112	AssertRCReturn(rc, rc);
5113	return VINF_SUCCESS;
5114	}
5115
5116	#ifdef VBOX_STRICT
5117	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5118	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5119	{
5120	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5121	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5122	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5123	}
5124	#endif
5125
5126	bool fSteppingDB = false;
5127	bool fInterceptMovDRx = false;
5128	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5129	if (pVCpu->hm.s.fSingleInstruction)
5130	{
5131	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5132	PVM pVM = pVCpu->CTX_SUFF(pVM);
5133	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5134	{
5135	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5136	Assert(fSteppingDB == false);
5137	}
5138	else
5139	{
5140	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5141	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5142	pVCpu->hm.s.fClearTrapFlag = true;
5143	fSteppingDB = true;
5144	}
5145	}
5146
5147	uint32_t u32GuestDr7;
5148	if ( fSteppingDB
5149	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5150	{
5151	/*
5152	* Use the combined guest and host DRx values found in the hypervisor register set
5153	* because the hypervisor debugger has breakpoints active or someone is single stepping
5154	* on the host side without a monitor trap flag.
5155	*
5156	* Note! DBGF expects a clean DR6 state before executing guest code.
5157	*/
5158	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5159	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5160	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5161	{
5162	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5163	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5164	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5165	}
5166	else
5167	#endif
5168	if (!CPUMIsHyperDebugStateActive(pVCpu))
5169	{
5170	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5171	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5172	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5173	}
5174
5175	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5176	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5177	pVCpu->hm.s.fUsingHyperDR7 = true;
5178	fInterceptMovDRx = true;
5179	}
5180	else
5181	{
5182	/*
5183	* If the guest has enabled debug registers, we need to load them prior to
5184	* executing guest code so they'll trigger at the right time.
5185	*/
5186	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5187	{
5188	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5189	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5190	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5191	{
5192	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5193	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5194	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5195	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5196	}
5197	else
5198	#endif
5199	if (!CPUMIsGuestDebugStateActive(pVCpu))
5200	{
5201	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5202	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5203	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5204	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5205	}
5206	Assert(!fInterceptMovDRx);
5207	}
5208	/*
5209	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5210	* must intercept #DB in order to maintain a correct DR6 guest value, and
5211	* because we need to intercept it to prevent nested #DBs from hanging the
5212	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5213	*/
5214	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5215	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5216	&& !CPUMIsGuestDebugStateActive(pVCpu))
5217	#else
5218	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5219	#endif
5220	{
5221	fInterceptMovDRx = true;
5222	}
5223
5224	/* Update DR7 with the actual guest value. */
5225	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5226	pVCpu->hm.s.fUsingHyperDR7 = false;
5227	}
5228
5229	if (fInterceptMovDRx)
5230	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5231	else
5232	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5233
5234	/*
5235	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5236	* monitor-trap flag and update our cache.
5237	*/
5238	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5239	{
5240	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5241	AssertRCReturn(rc2, rc2);
5242	pVmcsInfo->u32ProcCtls = uProcCtls;
5243	}
5244
5245	/*
5246	* Update guest DR7.
5247	*/
5248	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5249	AssertRCReturn(rc, rc);
5250
5251	/*
5252	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5253	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5254	*
5255	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5256	*/
5257	if (fSteppingDB)
5258	{
5259	Assert(pVCpu->hm.s.fSingleInstruction);
5260	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5261
5262	uint32_t fIntrState = 0;
5263	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5264	AssertRCReturn(rc, rc);
5265
5266	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5267	{
5268	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5269	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5270	AssertRCReturn(rc, rc);
5271	}
5272	}
5273
5274	return VINF_SUCCESS;
5275	}
5276
5277
5278	#ifdef VBOX_STRICT
5279	/**
5280	* Strict function to validate segment registers.
5281	*
5282	* @param pVCpu The cross context virtual CPU structure.
5283	* @param pVmcsInfo The VMCS info. object.
5284	*
5285	* @remarks Will import guest CR0 on strict builds during validation of
5286	* segments.
5287	*/
5288	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5289	{
5290	/*
5291	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5292	*
5293	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5294	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5295	* unusable bit and doesn't change the guest-context value.
5296	*/
5297	PVM pVM = pVCpu->CTX_SUFF(pVM);
5298	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5299	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5300	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5301	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5302	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5303	{
5304	/* Protected mode checks */
5305	/* CS */
5306	Assert(pCtx->cs.Attr.n.u1Present);
5307	Assert(!(pCtx->cs.Attr.u & 0xf00));
5308	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5309	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5310	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5311	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5312	\|\| (pCtx->cs.Attr.n.u1Granularity));
5313	/* CS cannot be loaded with NULL in protected mode. */
5314	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5315	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5316	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5317	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5318	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5319	else
5320	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5321	/* SS */
5322	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5323	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5324	if ( !(pCtx->cr0 & X86_CR0_PE)
5325	\|\| pCtx->cs.Attr.n.u4Type == 3)
5326	{
5327	Assert(!pCtx->ss.Attr.n.u2Dpl);
5328	}
5329	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5330	{
5331	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5332	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5333	Assert(pCtx->ss.Attr.n.u1Present);
5334	Assert(!(pCtx->ss.Attr.u & 0xf00));
5335	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5336	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5337	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5338	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5339	\|\| (pCtx->ss.Attr.n.u1Granularity));
5340	}
5341	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5342	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5343	{
5344	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5345	Assert(pCtx->ds.Attr.n.u1Present);
5346	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5347	Assert(!(pCtx->ds.Attr.u & 0xf00));
5348	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5349	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5350	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5351	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5352	\|\| (pCtx->ds.Attr.n.u1Granularity));
5353	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5354	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5355	}
5356	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5357	{
5358	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5359	Assert(pCtx->es.Attr.n.u1Present);
5360	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5361	Assert(!(pCtx->es.Attr.u & 0xf00));
5362	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5363	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5364	\|\| !(pCtx->es.Attr.n.u1Granularity));
5365	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5366	\|\| (pCtx->es.Attr.n.u1Granularity));
5367	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5368	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5369	}
5370	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5371	{
5372	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5373	Assert(pCtx->fs.Attr.n.u1Present);
5374	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5375	Assert(!(pCtx->fs.Attr.u & 0xf00));
5376	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5377	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5378	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5379	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5380	\|\| (pCtx->fs.Attr.n.u1Granularity));
5381	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5382	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5383	}
5384	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5385	{
5386	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5387	Assert(pCtx->gs.Attr.n.u1Present);
5388	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5389	Assert(!(pCtx->gs.Attr.u & 0xf00));
5390	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5391	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5392	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5393	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5394	\|\| (pCtx->gs.Attr.n.u1Granularity));
5395	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5396	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5397	}
5398	/* 64-bit capable CPUs. */
5399	# if HC_ARCH_BITS == 64
5400	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5401	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5402	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5403	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5404	# endif
5405	}
5406	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5407	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5408	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5409	{
5410	/* Real and v86 mode checks. */
5411	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5412	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5413	if (pVmcsInfo->RealMode.fRealOnV86Active)
5414	{
5415	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5416	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5417	}
5418	else
5419	{
5420	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5421	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5422	}
5423
5424	/* CS */
5425	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5426	Assert(pCtx->cs.u32Limit == 0xffff);
5427	Assert(u32CSAttr == 0xf3);
5428	/* SS */
5429	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5430	Assert(pCtx->ss.u32Limit == 0xffff);
5431	Assert(u32SSAttr == 0xf3);
5432	/* DS */
5433	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5434	Assert(pCtx->ds.u32Limit == 0xffff);
5435	Assert(u32DSAttr == 0xf3);
5436	/* ES */
5437	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5438	Assert(pCtx->es.u32Limit == 0xffff);
5439	Assert(u32ESAttr == 0xf3);
5440	/* FS */
5441	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5442	Assert(pCtx->fs.u32Limit == 0xffff);
5443	Assert(u32FSAttr == 0xf3);
5444	/* GS */
5445	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5446	Assert(pCtx->gs.u32Limit == 0xffff);
5447	Assert(u32GSAttr == 0xf3);
5448	/* 64-bit capable CPUs. */
5449	# if HC_ARCH_BITS == 64
5450	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5451	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5452	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5453	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5454	# endif
5455	}
5456	}
5457	#endif /* VBOX_STRICT */
5458
5459
5460	/**
5461	* Exports a guest segment register into the guest-state area in the VMCS.
5462	*
5463	* @returns VBox status code.
5464	* @param pVCpu The cross context virtual CPU structure.
5465	* @param pVmcsInfo The VMCS info. object.
5466	* @param iSegReg The segment register number (X86_SREG_XXX).
5467	* @param pSelReg Pointer to the segment selector.
5468	*
5469	* @remarks No-long-jump zone!!!
5470	*/
5471	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5472	{
5473	Assert(iSegReg < X86_SREG_COUNT);
5474	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5475	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5476	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5477	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5478
5479	uint32_t u32Access = pSelReg->Attr.u;
5480	if (pVmcsInfo->RealMode.fRealOnV86Active)
5481	{
5482	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5483	u32Access = 0xf3;
5484	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5485	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5486	RT_NOREF_PV(pVCpu);
5487	}
5488	else
5489	{
5490	/*
5491	* The way to differentiate between whether this is really a null selector or was just
5492	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5493	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5494	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5495	* NULL selectors loaded in protected-mode have their attribute as 0.
5496	*/
5497	if (!u32Access)
5498	u32Access = X86DESCATTR_UNUSABLE;
5499	}
5500
5501	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5502	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5503	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5504
5505	/*
5506	* Commit it to the VMCS.
5507	*/
5508	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5509	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5510	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5511	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5512	AssertRCReturn(rc, rc);
5513	return rc;
5514	}
5515
5516
5517	/**
5518	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5519	* area in the VMCS.
5520	*
5521	* @returns VBox status code.
5522	* @param pVCpu The cross context virtual CPU structure.
5523	* @param pVmxTransient The VMX-transient structure.
5524	*
5525	* @remarks Will import guest CR0 on strict builds during validation of
5526	* segments.
5527	* @remarks No-long-jump zone!!!
5528	*/
5529	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5530	{
5531	int rc = VERR_INTERNAL_ERROR_5;
5532	PVM pVM = pVCpu->CTX_SUFF(pVM);
5533	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5534	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5535
5536	/*
5537	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5538	*/
5539	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5540	{
5541	#ifdef VBOX_WITH_REM
5542	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5543	{
5544	Assert(!pVmxTransient->fIsNestedGuest);
5545	Assert(pVM->hm.s.vmx.pRealModeTSS);
5546	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5547	if ( pVmcsInfo->fWasInRealMode
5548	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5549	{
5550	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5551	in real-mode (e.g. OpenBSD 4.0) */
5552	REMFlushTBs(pVM);
5553	Log4Func(("Switch to protected mode detected!\n"));
5554	pVmcsInfo->fWasInRealMode = false;
5555	}
5556	}
5557	#endif
5558	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5559	{
5560	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5561	if (pVmcsInfo->RealMode.fRealOnV86Active)
5562	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5563	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5564	AssertRCReturn(rc, rc);
5565	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5566	}
5567
5568	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5569	{
5570	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5571	if (pVmcsInfo->RealMode.fRealOnV86Active)
5572	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5573	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5574	AssertRCReturn(rc, rc);
5575	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5576	}
5577
5578	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5579	{
5580	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5581	if (pVmcsInfo->RealMode.fRealOnV86Active)
5582	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5583	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5584	AssertRCReturn(rc, rc);
5585	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5586	}
5587
5588	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5589	{
5590	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5591	if (pVmcsInfo->RealMode.fRealOnV86Active)
5592	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5593	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5594	AssertRCReturn(rc, rc);
5595	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5596	}
5597
5598	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5599	{
5600	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5601	if (pVmcsInfo->RealMode.fRealOnV86Active)
5602	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5603	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5604	AssertRCReturn(rc, rc);
5605	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5606	}
5607
5608	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5609	{
5610	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5611	if (pVmcsInfo->RealMode.fRealOnV86Active)
5612	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5613	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5614	AssertRCReturn(rc, rc);
5615	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5616	}
5617
5618	#ifdef VBOX_STRICT
5619	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5620	#endif
5621	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5622	pCtx->cs.Attr.u));
5623	}
5624
5625	/*
5626	* Guest TR.
5627	*/
5628	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5629	{
5630	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5631
5632	/*
5633	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5634	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5635	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5636	*/
5637	uint16_t u16Sel;
5638	uint32_t u32Limit;
5639	uint64_t u64Base;
5640	uint32_t u32AccessRights;
5641	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5642	{
5643	u16Sel = pCtx->tr.Sel;
5644	u32Limit = pCtx->tr.u32Limit;
5645	u64Base = pCtx->tr.u64Base;
5646	u32AccessRights = pCtx->tr.Attr.u;
5647	}
5648	else
5649	{
5650	Assert(!pVmxTransient->fIsNestedGuest);
5651	Assert(pVM->hm.s.vmx.pRealModeTSS);
5652	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5653
5654	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5655	RTGCPHYS GCPhys;
5656	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5657	AssertRCReturn(rc, rc);
5658
5659	X86DESCATTR DescAttr;
5660	DescAttr.u = 0;
5661	DescAttr.n.u1Present = 1;
5662	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5663
5664	u16Sel = 0;
5665	u32Limit = HM_VTX_TSS_SIZE;
5666	u64Base = GCPhys;
5667	u32AccessRights = DescAttr.u;
5668	}
5669
5670	/* Validate. */
5671	Assert(!(u16Sel & RT_BIT(2)));
5672	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5673	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5674	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5675	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5676	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5677	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5678	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5679	Assert( (u32Limit & 0xfff) == 0xfff
5680	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5681	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5682	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5683
5684	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5685	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5686	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5687	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5688	AssertRCReturn(rc, rc);
5689
5690	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5691	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5692	}
5693
5694	/*
5695	* Guest GDTR.
5696	*/
5697	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5698	{
5699	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5700
5701	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5702	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5703	AssertRCReturn(rc, rc);
5704
5705	/* Validate. */
5706	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5707
5708	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5709	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5710	}
5711
5712	/*
5713	* Guest LDTR.
5714	*/
5715	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5716	{
5717	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5718
5719	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5720	uint32_t u32Access;
5721	if ( !pVmxTransient->fIsNestedGuest
5722	&& !pCtx->ldtr.Attr.u)
5723	u32Access = X86DESCATTR_UNUSABLE;
5724	else
5725	u32Access = pCtx->ldtr.Attr.u;
5726
5727	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5728	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5729	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5730	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5731	AssertRCReturn(rc, rc);
5732
5733	/* Validate. */
5734	if (!(u32Access & X86DESCATTR_UNUSABLE))
5735	{
5736	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5737	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5738	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5739	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5740	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5741	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5742	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5743	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5744	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5745	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5746	}
5747
5748	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5749	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5750	}
5751
5752	/*
5753	* Guest IDTR.
5754	*/
5755	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5756	{
5757	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5758
5759	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5760	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5761	AssertRCReturn(rc, rc);
5762
5763	/* Validate. */
5764	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5765
5766	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5767	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5768	}
5769
5770	return VINF_SUCCESS;
5771	}
5772
5773
5774	/**
5775	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5776	* areas.
5777	*
5778	* These MSRs will automatically be loaded to the host CPU on every successful
5779	* VM-entry and stored from the host CPU on every successful VM-exit.
5780	*
5781	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5782	* actual host MSR values are not- updated here for performance reasons. See
5783	* hmR0VmxExportHostMsrs().
5784	*
5785	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5786	*
5787	* @returns VBox status code.
5788	* @param pVCpu The cross context virtual CPU structure.
5789	* @param pVmxTransient The VMX-transient structure.
5790	*
5791	* @remarks No-long-jump zone!!!
5792	*/
5793	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5794	{
5795	AssertPtr(pVCpu);
5796	AssertPtr(pVmxTransient);
5797
5798	PVM pVM = pVCpu->CTX_SUFF(pVM);
5799	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5800
5801	/*
5802	* MSRs that we use the auto-load/store MSR area in the VMCS.
5803	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5804	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5805	*
5806	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5807	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5808	* emulation, nothing to do here.
5809	*/
5810	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5811	{
5812	if ( !pVmxTransient->fIsNestedGuest
5813	&& pVM->hm.s.fAllow64BitGuests)
5814	{
5815	#if HC_ARCH_BITS == 32
5816	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5817	Assert(!pVmxTransient->fIsNestedGuest);
5818
5819	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5820	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5821	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5822	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5823	AssertRCReturn(rc, rc);
5824	#endif
5825	}
5826	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5827	}
5828
5829	/*
5830	* Guest Sysenter MSRs.
5831	*/
5832	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5833	{
5834	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5835
5836	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5837	{
5838	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5839	AssertRCReturn(rc, rc);
5840	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5841	}
5842
5843	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5844	{
5845	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5846	AssertRCReturn(rc, rc);
5847	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5848	}
5849
5850	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5851	{
5852	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5853	AssertRCReturn(rc, rc);
5854	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5855	}
5856	}
5857
5858	/*
5859	* Guest/host EFER MSR.
5860	*/
5861	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5862	{
5863	/* Whether we are using the VMCS to swap the EFER MSR must have been
5864	determined earlier while exporting VM-entry/VM-exit controls. */
5865	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5866	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5867
5868	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5869	{
5870	/*
5871	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5872	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5873	*/
5874	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5875	{
5876	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5877	AssertRCReturn(rc, rc);
5878	}
5879	else
5880	{
5881	/*
5882	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5883	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5884	*/
5885	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5886	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5887	AssertRCReturn(rc, rc);
5888	}
5889	}
5890	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5891	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5892
5893	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5894	}
5895
5896	/*
5897	* Other MSRs.
5898	* Speculation Control (R/W).
5899	*/
5900	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5901	{
5902	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5903	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5904	{
5905	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5906	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5907	AssertRCReturn(rc, rc);
5908	}
5909	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5910	}
5911
5912	return VINF_SUCCESS;
5913	}
5914
5915
5916	/**
5917	* Selects up the appropriate function to run guest code.
5918	*
5919	* @returns VBox status code.
5920	* @param pVCpu The cross context virtual CPU structure.
5921	* @param pVmxTransient The VMX-transient structure.
5922	*
5923	* @remarks No-long-jump zone!!!
5924	*/
5925	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5926	{
5927	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5928	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5929
5930	if (CPUMIsGuestInLongModeEx(pCtx))
5931	{
5932	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5933	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5934	#endif
5935	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5936	#if HC_ARCH_BITS == 32
5937	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5938	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5939	{
5940	#ifdef VBOX_STRICT
5941	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5942	{
5943	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5944	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5945	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5946	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5947	("fCtxChanged=%#RX64\n", fCtxChanged));
5948	}
5949	#endif
5950	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5951
5952	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5953	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5954	pVmcsInfo->fSwitchedTo64on32 = true;
5955	Log4Func(("Selected 64-bit switcher\n"));
5956	}
5957	#else
5958	/* 64-bit host. */
5959	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5960	#endif
5961	}
5962	else
5963	{
5964	/* Guest is not in long mode, use the 32-bit handler. */
5965	#if HC_ARCH_BITS == 32
5966	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5967	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5968	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5969	{
5970	# ifdef VBOX_STRICT
5971	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5972	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5973	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5974	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5975	("fCtxChanged=%#RX64\n", fCtxChanged));
5976	# endif
5977	}
5978	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5979	/*
5980	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5981	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5982	* switcher flag now because we know the guest is in a sane state where it's safe
5983	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5984	* the much faster 32-bit switcher again.
5985	*/
5986	if (!pVmcsInfo->fSwitchedTo64on32)
5987	{
5988	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5989	Log4Func(("Selected 32-bit switcher\n"));
5990	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5991	}
5992	else
5993	{
5994	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5995	if ( pVmcsInfo->RealMode.fRealOnV86Active
5996	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5997	{
5998	pVmcsInfo->fSwitchedTo64on32 = false;
5999	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6000	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6001	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6002	\| HM_CHANGED_HOST_CONTEXT);
6003	Log4Func(("Selected 32-bit switcher (safe)\n"));
6004	}
6005	}
6006	# else
6007	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6008	# endif
6009	#else
6010	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6011	#endif
6012	}
6013	Assert(pVmcsInfo->pfnStartVM);
6014	return VINF_SUCCESS;
6015	}
6016
6017
6018	/**
6019	* Wrapper for running the guest code in VT-x.
6020	*
6021	* @returns VBox status code, no informational status codes.
6022	* @param pVCpu The cross context virtual CPU structure.
6023	* @param pVmxTransient The VMX-transient structure.
6024	*
6025	* @remarks No-long-jump zone!!!
6026	*/
6027	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6028	{
6029	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6030	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6031	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6032
6033	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6034
6035	/*
6036	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6037	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6038	* callee-saved and thus the need for this XMM wrapper.
6039	*
6040	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6041	*/
6042	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6043	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6044	PVM pVM = pVCpu->CTX_SUFF(pVM);
6045	#ifdef VBOX_WITH_KERNEL_USING_XMM
6046	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6047	#else
6048	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6049	#endif
6050	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6051	return rc;
6052	}
6053
6054
6055	/**
6056	* Reports world-switch error and dumps some useful debug info.
6057	*
6058	* @param pVCpu The cross context virtual CPU structure.
6059	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6060	* @param pVmxTransient The VMX-transient structure (only
6061	* exitReason updated).
6062	*/
6063	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6064	{
6065	Assert(pVCpu);
6066	Assert(pVmxTransient);
6067	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6068
6069	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6070	switch (rcVMRun)
6071	{
6072	case VERR_VMX_INVALID_VMXON_PTR:
6073	AssertFailed();
6074	break;
6075	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6076	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6077	{
6078	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6079	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6080	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6081	AssertRC(rc);
6082
6083	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6084	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6085	Cannot do it here as we may have been long preempted. */
6086
6087	#ifdef VBOX_STRICT
6088	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6089	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6090	pVmxTransient->uExitReason));
6091	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6092	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6093	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6094	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6095	else
6096	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6097	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6098	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6099
6100	/* VMX control bits. */
6101	uint32_t u32Val;
6102	uint64_t u64Val;
6103	RTHCUINTREG uHCReg;
6104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6105	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6107	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6108	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6109	{
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6112	}
6113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6114	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6116	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6118	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6120	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6122	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6124	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6126	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6128	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6129	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6130	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6131	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6132	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6133	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6134	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6135	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6136	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6137	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6138	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6139	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6140	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6141	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6142	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6143	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6144	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6145	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6146	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6147	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6148	{
6149	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6150	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6151	}
6152
6153	/* Guest bits. */
6154	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6155	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6156	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6157	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6158	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6159	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6160	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6161	{
6162	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6163	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6164	}
6165
6166	/* Host bits. */
6167	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6168	Log4(("Host CR0 %#RHr\n", uHCReg));
6169	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6170	Log4(("Host CR3 %#RHr\n", uHCReg));
6171	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6172	Log4(("Host CR4 %#RHr\n", uHCReg));
6173
6174	RTGDTR HostGdtr;
6175	PCX86DESCHC pDesc;
6176	ASMGetGDTR(&HostGdtr);
6177	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6178	Log4(("Host CS %#08x\n", u32Val));
6179	if (u32Val < HostGdtr.cbGdt)
6180	{
6181	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6182	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6183	}
6184
6185	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6186	Log4(("Host DS %#08x\n", u32Val));
6187	if (u32Val < HostGdtr.cbGdt)
6188	{
6189	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6190	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6191	}
6192
6193	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6194	Log4(("Host ES %#08x\n", u32Val));
6195	if (u32Val < HostGdtr.cbGdt)
6196	{
6197	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6198	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6199	}
6200
6201	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6202	Log4(("Host FS %#08x\n", u32Val));
6203	if (u32Val < HostGdtr.cbGdt)
6204	{
6205	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6206	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6207	}
6208
6209	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6210	Log4(("Host GS %#08x\n", u32Val));
6211	if (u32Val < HostGdtr.cbGdt)
6212	{
6213	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6214	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6215	}
6216
6217	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6218	Log4(("Host SS %#08x\n", u32Val));
6219	if (u32Val < HostGdtr.cbGdt)
6220	{
6221	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6222	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6223	}
6224
6225	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6226	Log4(("Host TR %#08x\n", u32Val));
6227	if (u32Val < HostGdtr.cbGdt)
6228	{
6229	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6230	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6231	}
6232
6233	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6234	Log4(("Host TR Base %#RHv\n", uHCReg));
6235	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6236	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6237	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6238	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6239	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6240	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6241	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6242	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6243	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6244	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6245	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6246	Log4(("Host RSP %#RHv\n", uHCReg));
6247	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6248	Log4(("Host RIP %#RHv\n", uHCReg));
6249	# if HC_ARCH_BITS == 64
6250	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6251	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6252	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6253	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6254	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6255	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6256	# endif
6257	#endif /* VBOX_STRICT */
6258	break;
6259	}
6260
6261	default:
6262	/* Impossible */
6263	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6264	break;
6265	}
6266	}
6267
6268
6269	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6270	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6271	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6272	# endif
6273
6274	/**
6275	* Initialize the VMCS-Read cache.
6276	*
6277	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6278	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6279	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6280	* (those that have a 32-bit FULL & HIGH part).
6281	*
6282	* @param pVCpu The cross context virtual CPU structure.
6283	*/
6284	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6285	{
6286	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6287	do { \
6288	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6289	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6290	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6291	++cReadFields; \
6292	} while (0)
6293
6294	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6295	uint32_t cReadFields = 0;
6296
6297	/*
6298	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6299	* and serve to indicate exceptions to the rules.
6300	*/
6301
6302	/* Guest-natural selector base fields. */
6303	#if 0
6304	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6307	#endif
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6320	#if 0
6321	/* Unused natural width guest-state fields. */
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6324	#endif
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6327
6328	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6329	these 64-bit fields (using "FULL" and "HIGH" fields). */
6330	#if 0
6331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6334	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6335	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6336	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6337	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6338	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6339	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6340	#endif
6341
6342	/* Natural width guest-state fields. */
6343	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6344	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6345
6346	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6347	{
6348	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6349	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6350	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6351	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6352	}
6353	else
6354	{
6355	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6356	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6357	}
6358
6359	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6360	}
6361
6362
6363	/**
6364	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6365	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6366	* darwin, running 64-bit guests).
6367	*
6368	* @returns VBox status code.
6369	* @param pVCpu The cross context virtual CPU structure.
6370	* @param idxField The VMCS field encoding.
6371	* @param u64Val 16, 32 or 64-bit value.
6372	*/
6373	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6374	{
6375	int rc;
6376	switch (idxField)
6377	{
6378	/*
6379	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6380	*/
6381	/* 64-bit Control fields. */
6382	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6383	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6384	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6385	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6386	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6387	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6388	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6389	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6390	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6391	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6392	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6393	case VMX_VMCS64_CTRL_EPTP_FULL:
6394	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6395	/* 64-bit Guest-state fields. */
6396	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6397	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6398	case VMX_VMCS64_GUEST_PAT_FULL:
6399	case VMX_VMCS64_GUEST_EFER_FULL:
6400	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6401	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6402	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6403	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6404	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6405	/* 64-bit Host-state fields. */
6406	case VMX_VMCS64_HOST_PAT_FULL:
6407	case VMX_VMCS64_HOST_EFER_FULL:
6408	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6409	{
6410	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6411	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6412	break;
6413	}
6414
6415	/*
6416	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6417	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6418	*/
6419	/* Natural-width Guest-state fields. */
6420	case VMX_VMCS_GUEST_CR3:
6421	case VMX_VMCS_GUEST_ES_BASE:
6422	case VMX_VMCS_GUEST_CS_BASE:
6423	case VMX_VMCS_GUEST_SS_BASE:
6424	case VMX_VMCS_GUEST_DS_BASE:
6425	case VMX_VMCS_GUEST_FS_BASE:
6426	case VMX_VMCS_GUEST_GS_BASE:
6427	case VMX_VMCS_GUEST_LDTR_BASE:
6428	case VMX_VMCS_GUEST_TR_BASE:
6429	case VMX_VMCS_GUEST_GDTR_BASE:
6430	case VMX_VMCS_GUEST_IDTR_BASE:
6431	case VMX_VMCS_GUEST_RSP:
6432	case VMX_VMCS_GUEST_RIP:
6433	case VMX_VMCS_GUEST_SYSENTER_ESP:
6434	case VMX_VMCS_GUEST_SYSENTER_EIP:
6435	{
6436	if (!(RT_HI_U32(u64Val)))
6437	{
6438	/* If this field is 64-bit, VT-x will zero out the top bits. */
6439	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6440	}
6441	else
6442	{
6443	/* Assert that only the 32->64 switcher case should ever come here. */
6444	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6445	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6446	}
6447	break;
6448	}
6449
6450	default:
6451	{
6452	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6453	pVCpu->hm.s.u32HMError = idxField;
6454	rc = VERR_INVALID_PARAMETER;
6455	break;
6456	}
6457	}
6458	AssertRCReturn(rc, rc);
6459	return rc;
6460	}
6461
6462
6463	/**
6464	* Queue up a VMWRITE by using the VMCS write cache.
6465	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6466	*
6467	* @param pVCpu The cross context virtual CPU structure.
6468	* @param idxField The VMCS field encoding.
6469	* @param u64Val 16, 32 or 64-bit value.
6470	*/
6471	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6472	{
6473	AssertPtr(pVCpu);
6474	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6475
6476	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6477	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6478
6479	/* Make sure there are no duplicates. */
6480	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6481	{
6482	if (pCache->Write.aField[i] == idxField)
6483	{
6484	pCache->Write.aFieldVal[i] = u64Val;
6485	return VINF_SUCCESS;
6486	}
6487	}
6488
6489	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6490	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6491	pCache->Write.cValidEntries++;
6492	return VINF_SUCCESS;
6493	}
6494	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6495
6496
6497	/**
6498	* Sets up the usage of TSC-offsetting and updates the VMCS.
6499	*
6500	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6501	* VMX-preemption timer.
6502	*
6503	* @returns VBox status code.
6504	* @param pVCpu The cross context virtual CPU structure.
6505	* @param pVmxTransient The VMX-transient structure.
6506	*
6507	* @remarks No-long-jump zone!!!
6508	*/
6509	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6510	{
6511	bool fOffsettedTsc;
6512	bool fParavirtTsc;
6513	uint64_t uTscOffset;
6514	PVM pVM = pVCpu->CTX_SUFF(pVM);
6515	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6516
6517	if (pVM->hm.s.vmx.fUsePreemptTimer)
6518	{
6519	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6520
6521	/* Make sure the returned values have sane upper and lower boundaries. */
6522	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6523	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6524	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6525	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6526
6527	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6528	* preemption timers here. We probably need to clamp the preemption timer,
6529	* after converting the timer value to the host. */
6530	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6531	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6532	AssertRC(rc);
6533	}
6534	else
6535	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6536
6537	if (fParavirtTsc)
6538	{
6539	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6540	information before every VM-entry, hence disable it for performance sake. */
6541	#if 0
6542	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6543	AssertRC(rc);
6544	#endif
6545	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6546	}
6547
6548	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6549	if ( fOffsettedTsc
6550	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6551	{
6552	if (pVmxTransient->fIsNestedGuest)
6553	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6554	if (pVmcsInfo->u64TscOffset != uTscOffset)
6555	{
6556	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6557	AssertRC(rc);
6558	pVmcsInfo->u64TscOffset = uTscOffset;
6559	}
6560
6561	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6562	{
6563	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6564	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6565	AssertRC(rc);
6566	pVmcsInfo->u32ProcCtls = uProcCtls;
6567	}
6568	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6569	}
6570	else
6571	{
6572	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6573	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6574	{
6575	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6576	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6577	AssertRC(rc);
6578	pVmcsInfo->u32ProcCtls = uProcCtls;
6579	}
6580	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6581	}
6582	}
6583
6584
6585	/**
6586	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6587	* VM-exit interruption info type.
6588	*
6589	* @returns The IEM exception flags.
6590	* @param uVector The event vector.
6591	* @param uVmxEventType The VMX event type.
6592	*
6593	* @remarks This function currently only constructs flags required for
6594	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6595	* and CR2 aspects of an exception are not included).
6596	*/
6597	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6598	{
6599	uint32_t fIemXcptFlags;
6600	switch (uVmxEventType)
6601	{
6602	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6603	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6604	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6605	break;
6606
6607	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6608	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6609	break;
6610
6611	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6612	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6613	break;
6614
6615	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6616	{
6617	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6618	if (uVector == X86_XCPT_BP)
6619	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6620	else if (uVector == X86_XCPT_OF)
6621	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6622	else
6623	{
6624	fIemXcptFlags = 0;
6625	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6626	}
6627	break;
6628	}
6629
6630	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6631	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6632	break;
6633
6634	default:
6635	fIemXcptFlags = 0;
6636	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6637	break;
6638	}
6639	return fIemXcptFlags;
6640	}
6641
6642
6643	/**
6644	* Sets an event as a pending event to be injected into the guest.
6645	*
6646	* @param pVCpu The cross context virtual CPU structure.
6647	* @param u32IntInfo The VM-entry interruption-information field.
6648	* @param cbInstr The VM-entry instruction length in bytes (for software
6649	* interrupts, exceptions and privileged software
6650	* exceptions).
6651	* @param u32ErrCode The VM-entry exception error code.
6652	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6653	* page-fault.
6654	*/
6655	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6656	RTGCUINTPTR GCPtrFaultAddress)
6657	{
6658	Assert(!pVCpu->hm.s.Event.fPending);
6659	pVCpu->hm.s.Event.fPending = true;
6660	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6661	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6662	pVCpu->hm.s.Event.cbInstr = cbInstr;
6663	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6664	}
6665
6666
6667	/**
6668	* Sets an external interrupt as pending-for-injection into the VM.
6669	*
6670	* @param pVCpu The cross context virtual CPU structure.
6671	* @param u8Interrupt The external interrupt vector.
6672	*/
6673	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6674	{
6675	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6676	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6677	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6678	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6679	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6680	}
6681
6682
6683	/**
6684	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6685	*
6686	* @param pVCpu The cross context virtual CPU structure.
6687	*/
6688	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6689	{
6690	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6691	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6692	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6693	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6694	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6695	}
6696
6697
6698	/**
6699	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6700	*
6701	* @param pVCpu The cross context virtual CPU structure.
6702	*/
6703	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6704	{
6705	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6706	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6707	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6708	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6709	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6710	}
6711
6712
6713	/**
6714	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6715	*
6716	* @param pVCpu The cross context virtual CPU structure.
6717	*/
6718	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6719	{
6720	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6721	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6722	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6723	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6724	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6725	}
6726
6727
6728	/**
6729	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6730	*
6731	* @param pVCpu The cross context virtual CPU structure.
6732	*/
6733	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6734	{
6735	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6736	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6737	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6738	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6739	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6740	}
6741
6742
6743	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6744	/**
6745	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6746	*
6747	* @param pVCpu The cross context virtual CPU structure.
6748	* @param u32ErrCode The error code for the general-protection exception.
6749	*/
6750	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6751	{
6752	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6753	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6754	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6755	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6756	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6757	}
6758
6759
6760	/**
6761	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6762	*
6763	* @param pVCpu The cross context virtual CPU structure.
6764	* @param u32ErrCode The error code for the stack exception.
6765	*/
6766	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6767	{
6768	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6769	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6770	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6771	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6772	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6773	}
6774
6775
6776	/**
6777	* Decodes the memory operand of an instruction that caused a VM-exit.
6778	*
6779	* The VM-exit qualification field provides the displacement field for memory
6780	* operand instructions, if any.
6781	*
6782	* @returns Strict VBox status code (i.e. informational status codes too).
6783	* @retval VINF_SUCCESS if the operand was successfully decoded.
6784	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6785	* operand.
6786	* @param pVCpu The cross context virtual CPU structure.
6787	* @param uExitInstrInfo The VM-exit instruction information field.
6788	* @param enmMemAccess The memory operand's access type (read or write).
6789	* @param GCPtrDisp The instruction displacement field, if any. For
6790	* RIP-relative addressing pass RIP + displacement here.
6791	* @param pGCPtrMem Where to store the effective destination memory address.
6792	*
6793	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6794	* virtual-8086 mode hence skips those checks while verifying if the
6795	* segment is valid.
6796	*/
6797	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6798	PRTGCPTR pGCPtrMem)
6799	{
6800	Assert(pGCPtrMem);
6801	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6802	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6803	\| CPUMCTX_EXTRN_CR0);
6804
6805	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6806	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6807	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6808
6809	VMXEXITINSTRINFO ExitInstrInfo;
6810	ExitInstrInfo.u = uExitInstrInfo;
6811	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6812	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6813	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6814	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6815	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6816	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6817	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6818	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6819	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6820
6821	/*
6822	* Validate instruction information.
6823	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6824	*/
6825	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6826	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6827	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6828	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6829	AssertLogRelMsgReturn(fIsMemOperand,
6830	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6831
6832	/*
6833	* Compute the complete effective address.
6834	*
6835	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6836	* See AMD spec. 4.5.2 "Segment Registers".
6837	*/
6838	RTGCPTR GCPtrMem = GCPtrDisp;
6839	if (fBaseRegValid)
6840	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6841	if (fIdxRegValid)
6842	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6843
6844	RTGCPTR const GCPtrOff = GCPtrMem;
6845	if ( !fIsLongMode
6846	\|\| iSegReg >= X86_SREG_FS)
6847	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6848	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6849
6850	/*
6851	* Validate effective address.
6852	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6853	*/
6854	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6855	Assert(cbAccess > 0);
6856	if (fIsLongMode)
6857	{
6858	if (X86_IS_CANONICAL(GCPtrMem))
6859	{
6860	*pGCPtrMem = GCPtrMem;
6861	return VINF_SUCCESS;
6862	}
6863
6864	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6865	* "Data Limit Checks in 64-bit Mode". */
6866	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6867	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6868	return VINF_HM_PENDING_XCPT;
6869	}
6870
6871	/*
6872	* This is a watered down version of iemMemApplySegment().
6873	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6874	* and segment CPL/DPL checks are skipped.
6875	*/
6876	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6877	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6878	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6879
6880	/* Check if the segment is present and usable. */
6881	if ( pSel->Attr.n.u1Present
6882	&& !pSel->Attr.n.u1Unusable)
6883	{
6884	Assert(pSel->Attr.n.u1DescType);
6885	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6886	{
6887	/* Check permissions for the data segment. */
6888	if ( enmMemAccess == VMXMEMACCESS_WRITE
6889	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6890	{
6891	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6892	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6893	return VINF_HM_PENDING_XCPT;
6894	}
6895
6896	/* Check limits if it's a normal data segment. */
6897	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6898	{
6899	if ( GCPtrFirst32 > pSel->u32Limit
6900	\|\| GCPtrLast32 > pSel->u32Limit)
6901	{
6902	Log4Func(("Data segment limit exceeded."
6903	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6904	GCPtrLast32, pSel->u32Limit));
6905	if (iSegReg == X86_SREG_SS)
6906	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6907	else
6908	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6909	return VINF_HM_PENDING_XCPT;
6910	}
6911	}
6912	else
6913	{
6914	/* Check limits if it's an expand-down data segment.
6915	Note! The upper boundary is defined by the B bit, not the G bit! */
6916	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6917	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6918	{
6919	Log4Func(("Expand-down data segment limit exceeded."
6920	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6921	GCPtrLast32, pSel->u32Limit));
6922	if (iSegReg == X86_SREG_SS)
6923	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6924	else
6925	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6926	return VINF_HM_PENDING_XCPT;
6927	}
6928	}
6929	}
6930	else
6931	{
6932	/* Check permissions for the code segment. */
6933	if ( enmMemAccess == VMXMEMACCESS_WRITE
6934	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6935	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6936	{
6937	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6938	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6939	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6940	return VINF_HM_PENDING_XCPT;
6941	}
6942
6943	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6944	if ( GCPtrFirst32 > pSel->u32Limit
6945	\|\| GCPtrLast32 > pSel->u32Limit)
6946	{
6947	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6948	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6949	if (iSegReg == X86_SREG_SS)
6950	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6951	else
6952	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6953	return VINF_HM_PENDING_XCPT;
6954	}
6955	}
6956	}
6957	else
6958	{
6959	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6960	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6961	return VINF_HM_PENDING_XCPT;
6962	}
6963
6964	*pGCPtrMem = GCPtrMem;
6965	return VINF_SUCCESS;
6966	}
6967
6968
6969	/**
6970	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6971	* guest attempting to execute a VMX instruction.
6972	*
6973	* @returns Strict VBox status code (i.e. informational status codes too).
6974	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6975	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6976	*
6977	* @param pVCpu The cross context virtual CPU structure.
6978	* @param uExitReason The VM-exit reason.
6979	*
6980	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6981	* @remarks No-long-jump zone!!!
6982	*/
6983	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6984	{
6985	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6986	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6987
6988	/*
6989	* The physical CPU would have already checked the CPU mode/code segment.
6990	* We shall just assert here for paranoia.
6991	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
6992	*/
6993	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6994	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6995	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
6996
6997	if (uExitReason == VMX_EXIT_VMXON)
6998	{
6999	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7000
7001	/*
7002	* We check CR4.VMXE because it is required to be always set while in VMX operation
7003	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
7004	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7005	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7006	*/
7007	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7008	{
7009	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7010	hmR0VmxSetPendingXcptUD(pVCpu);
7011	return VINF_HM_PENDING_XCPT;
7012	}
7013	}
7014	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7015	{
7016	/*
7017	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7018	* (other than VMXON), we need to raise a #UD.
7019	*/
7020	Log4Func(("Not in VMX root mode -> #UD\n"));
7021	hmR0VmxSetPendingXcptUD(pVCpu);
7022	return VINF_HM_PENDING_XCPT;
7023	}
7024
7025	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7026	return VINF_SUCCESS;
7027	}
7028	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7029
7030
7031	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7032	{
7033	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7034
7035	/*
7036	* If VT-x marks the segment as unusable, most other bits remain undefined:
7037	* - For CS the L, D and G bits have meaning.
7038	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7039	* - For the remaining data segments no bits are defined.
7040	*
7041	* The present bit and the unusable bit has been observed to be set at the
7042	* same time (the selector was supposed to be invalid as we started executing
7043	* a V8086 interrupt in ring-0).
7044	*
7045	* What should be important for the rest of the VBox code, is that the P bit is
7046	* cleared. Some of the other VBox code recognizes the unusable bit, but
7047	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7048	* safe side here, we'll strip off P and other bits we don't care about. If
7049	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7050	*
7051	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7052	*/
7053	#ifdef VBOX_STRICT
7054	uint32_t const uAttr = pSelReg->Attr.u;
7055	#endif
7056
7057	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7058	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7059	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7060
7061	#ifdef VBOX_STRICT
7062	VMMRZCallRing3Disable(pVCpu);
7063	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7064	# ifdef DEBUG_bird
7065	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7066	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7067	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7068	# endif
7069	VMMRZCallRing3Enable(pVCpu);
7070	NOREF(uAttr);
7071	#endif
7072	RT_NOREF2(pVCpu, idxSel);
7073	}
7074
7075
7076	/**
7077	* Imports a guest segment register from the current VMCS into the guest-CPU
7078	* context.
7079	*
7080	* @returns VBox status code.
7081	* @param pVCpu The cross context virtual CPU structure.
7082	* @param iSegReg The segment register number (X86_SREG_XXX).
7083	*
7084	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7085	* do not log!
7086	*/
7087	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7088	{
7089	Assert(iSegReg < X86_SREG_COUNT);
7090
7091	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7092	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7093	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7094	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7095	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7096	#else
7097	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7098	#endif
7099	uint64_t u64Base;
7100	uint32_t u32Sel, u32Limit, u32Attr;
7101	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7102	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7103	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7104	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7105	if (RT_SUCCESS(rc))
7106	{
7107	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7108	pSelReg->Sel = u32Sel;
7109	pSelReg->ValidSel = u32Sel;
7110	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7111	pSelReg->u32Limit = u32Limit;
7112	pSelReg->u64Base = u64Base;
7113	pSelReg->Attr.u = u32Attr;
7114	if (u32Attr & X86DESCATTR_UNUSABLE)
7115	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7116	}
7117	return rc;
7118	}
7119
7120
7121	/**
7122	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7123	*
7124	* @returns VBox status code.
7125	* @param pVCpu The cross context virtual CPU structure.
7126	*
7127	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7128	* do not log!
7129	*/
7130	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7131	{
7132	uint64_t u64Base;
7133	uint32_t u32Sel, u32Limit, u32Attr;
7134	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7135	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7136	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7137	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7138	if (RT_SUCCESS(rc))
7139	{
7140	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7141	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7142	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7143	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7144	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7145	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7146	if (u32Attr & X86DESCATTR_UNUSABLE)
7147	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7148	}
7149	return rc;
7150	}
7151
7152
7153	/**
7154	* Imports the guest TR from the current VMCS into the guest-CPU context.
7155	*
7156	* @returns VBox status code.
7157	* @param pVCpu The cross context virtual CPU structure.
7158	*
7159	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7160	* do not log!
7161	*/
7162	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7163	{
7164	uint32_t u32Sel, u32Limit, u32Attr;
7165	uint64_t u64Base;
7166	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7167	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7168	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7169	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7170	AssertRCReturn(rc, rc);
7171
7172	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7173	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7174	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7175	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7176	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7177	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7178	/* TR is the only selector that can never be unusable. */
7179	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7180	return VINF_SUCCESS;
7181	}
7182
7183
7184	/**
7185	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7186	*
7187	* @returns VBox status code.
7188	* @param pVCpu The cross context virtual CPU structure.
7189	*
7190	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7191	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7192	* instead!!!
7193	*/
7194	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7195	{
7196	uint64_t u64Val;
7197	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7198	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7199	{
7200	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7201	if (RT_SUCCESS(rc))
7202	{
7203	pCtx->rip = u64Val;
7204	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7205	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7206	}
7207	return rc;
7208	}
7209	return VINF_SUCCESS;
7210	}
7211
7212
7213	/**
7214	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7215	*
7216	* @returns VBox status code.
7217	* @param pVCpu The cross context virtual CPU structure.
7218	* @param pVmcsInfo The VMCS info. object.
7219	*
7220	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7221	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7222	* instead!!!
7223	*/
7224	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7225	{
7226	uint32_t u32Val;
7227	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7228	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7229	{
7230	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7231	if (RT_SUCCESS(rc))
7232	{
7233	pCtx->eflags.u32 = u32Val;
7234
7235	/* Restore eflags for real-on-v86-mode hack. */
7236	if (pVmcsInfo->RealMode.fRealOnV86Active)
7237	{
7238	pCtx->eflags.Bits.u1VM = 0;
7239	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7240	}
7241	}
7242	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7243	return rc;
7244	}
7245	return VINF_SUCCESS;
7246	}
7247
7248
7249	/**
7250	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7251	* context.
7252	*
7253	* @returns VBox status code.
7254	* @param pVCpu The cross context virtual CPU structure.
7255	* @param pVmcsInfo The VMCS info. object.
7256	*
7257	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7258	* do not log!
7259	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7260	* instead!!!
7261	*/
7262	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7263	{
7264	uint32_t u32Val;
7265	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7266	if (RT_SUCCESS(rc))
7267	{
7268	if (!u32Val)
7269	{
7270	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7271	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7272
7273	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7274	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7275	}
7276	else
7277	{
7278	/*
7279	* We must import RIP here to set our EM interrupt-inhibited state.
7280	* We also import RFLAGS as our code that evaluates pending interrupts
7281	* before VM-entry requires it.
7282	*/
7283	rc = hmR0VmxImportGuestRip(pVCpu);
7284	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7285	if (RT_SUCCESS(rc))
7286	{
7287	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7288	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7289	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7290	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7291
7292	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7293	{
7294	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7295	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7296	}
7297	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7298	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7299	}
7300	}
7301	}
7302	return rc;
7303	}
7304
7305
7306	/**
7307	* Worker for VMXR0ImportStateOnDemand.
7308	*
7309	* @returns VBox status code.
7310	* @param pVCpu The cross context virtual CPU structure.
7311	* @param pVmcsInfo The VMCS info. object.
7312	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7313	*/
7314	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7315	{
7316	#define VMXLOCAL_BREAK_RC(a_rc) \
7317	if (RT_SUCCESS(a_rc)) \
7318	{ } \
7319	else \
7320	break
7321
7322	int rc = VINF_SUCCESS;
7323	PVM pVM = pVCpu->CTX_SUFF(pVM);
7324	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7325	uint64_t u64Val;
7326	uint32_t u32Val;
7327
7328	/*
7329	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7330	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7331	* neither are other host platforms.
7332	*
7333	* Committing this temporarily as it prevents BSOD.
7334	*
7335	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7336	*/
7337	#ifdef RT_OS_WINDOWS
7338	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7339	return VERR_HM_IPE_1;
7340	#endif
7341
7342	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7343
7344	/*
7345	* We disable interrupts to make the updating of the state and in particular
7346	* the fExtrn modification atomic wrt to preemption hooks.
7347	*/
7348	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7349
7350	fWhat &= pCtx->fExtrn;
7351	if (fWhat)
7352	{
7353	do
7354	{
7355	if (fWhat & CPUMCTX_EXTRN_RIP)
7356	{
7357	rc = hmR0VmxImportGuestRip(pVCpu);
7358	VMXLOCAL_BREAK_RC(rc);
7359	}
7360
7361	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7362	{
7363	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7364	VMXLOCAL_BREAK_RC(rc);
7365	}
7366
7367	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7368	{
7369	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7370	VMXLOCAL_BREAK_RC(rc);
7371	}
7372
7373	if (fWhat & CPUMCTX_EXTRN_RSP)
7374	{
7375	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7376	VMXLOCAL_BREAK_RC(rc);
7377	pCtx->rsp = u64Val;
7378	}
7379
7380	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7381	{
7382	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7383	if (fWhat & CPUMCTX_EXTRN_CS)
7384	{
7385	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7386	rc \|= hmR0VmxImportGuestRip(pVCpu);
7387	if (fRealOnV86Active)
7388	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7389	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7390	}
7391	if (fWhat & CPUMCTX_EXTRN_SS)
7392	{
7393	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7394	if (fRealOnV86Active)
7395	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7396	}
7397	if (fWhat & CPUMCTX_EXTRN_DS)
7398	{
7399	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7400	if (fRealOnV86Active)
7401	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7402	}
7403	if (fWhat & CPUMCTX_EXTRN_ES)
7404	{
7405	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7406	if (fRealOnV86Active)
7407	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7408	}
7409	if (fWhat & CPUMCTX_EXTRN_FS)
7410	{
7411	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7412	if (fRealOnV86Active)
7413	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7414	}
7415	if (fWhat & CPUMCTX_EXTRN_GS)
7416	{
7417	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7418	if (fRealOnV86Active)
7419	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7420	}
7421	VMXLOCAL_BREAK_RC(rc);
7422	}
7423
7424	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7425	{
7426	if (fWhat & CPUMCTX_EXTRN_LDTR)
7427	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7428
7429	if (fWhat & CPUMCTX_EXTRN_GDTR)
7430	{
7431	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7432	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7433	pCtx->gdtr.pGdt = u64Val;
7434	pCtx->gdtr.cbGdt = u32Val;
7435	}
7436
7437	/* Guest IDTR. */
7438	if (fWhat & CPUMCTX_EXTRN_IDTR)
7439	{
7440	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7441	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7442	pCtx->idtr.pIdt = u64Val;
7443	pCtx->idtr.cbIdt = u32Val;
7444	}
7445
7446	/* Guest TR. */
7447	if (fWhat & CPUMCTX_EXTRN_TR)
7448	{
7449	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7450	don't need to import that one. */
7451	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7452	rc \|= hmR0VmxImportGuestTr(pVCpu);
7453	}
7454	VMXLOCAL_BREAK_RC(rc);
7455	}
7456
7457	if (fWhat & CPUMCTX_EXTRN_DR7)
7458	{
7459	if (!pVCpu->hm.s.fUsingHyperDR7)
7460	{
7461	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7462	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7463	VMXLOCAL_BREAK_RC(rc);
7464	pCtx->dr[7] = u32Val;
7465	}
7466	}
7467
7468	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7469	{
7470	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7471	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7472	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7473	pCtx->SysEnter.cs = u32Val;
7474	VMXLOCAL_BREAK_RC(rc);
7475	}
7476
7477	#if HC_ARCH_BITS == 64
7478	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7479	{
7480	if ( pVM->hm.s.fAllow64BitGuests
7481	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7482	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7483	}
7484
7485	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7486	{
7487	if ( pVM->hm.s.fAllow64BitGuests
7488	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7489	{
7490	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7491	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7492	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7493	}
7494	}
7495	#endif
7496
7497	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7498	#if HC_ARCH_BITS == 32
7499	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7500	#endif
7501	)
7502	{
7503	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7504	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7505	Assert(pMsrs);
7506	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7507	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7508	for (uint32_t i = 0; i < cMsrs; i++)
7509	{
7510	uint32_t const idMsr = pMsrs[i].u32Msr;
7511	switch (idMsr)
7512	{
7513	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7514	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7515	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7516	#if HC_ARCH_BITS == 32
7517	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7518	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7519	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7520	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7521	#endif
7522	default:
7523	{
7524	pCtx->fExtrn = 0;
7525	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7526	ASMSetFlags(fEFlags);
7527	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7528	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7529	}
7530	}
7531	}
7532	}
7533
7534	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7535	{
7536	uint64_t u64Shadow;
7537	if (fWhat & CPUMCTX_EXTRN_CR0)
7538	{
7539	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7540	* remove when we drop 32-bit host w/ 64-bit host support, see
7541	* @bugref{9180#c39}. */
7542	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7543	#if HC_ARCH_BITS == 32
7544	uint32_t u32Shadow;
7545	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7546	u64Shadow = u32Shadow;
7547	#else
7548	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7549	#endif
7550	VMXLOCAL_BREAK_RC(rc);
7551	u64Val = u32Val;
7552	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7553	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7554	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7555	CPUMSetGuestCR0(pVCpu, u64Val);
7556	VMMRZCallRing3Enable(pVCpu);
7557	}
7558
7559	if (fWhat & CPUMCTX_EXTRN_CR4)
7560	{
7561	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7562	* remove when we drop 32-bit host w/ 64-bit host support, see
7563	* @bugref{9180#c39}. */
7564	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7565	#if HC_ARCH_BITS == 32
7566	uint32_t u32Shadow;
7567	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7568	u64Shadow = u32Shadow;
7569	#else
7570	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7571	#endif
7572	VMXLOCAL_BREAK_RC(rc);
7573	u64Val = u32Val;
7574	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7575	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7576	pCtx->cr4 = u64Val;
7577	}
7578
7579	if (fWhat & CPUMCTX_EXTRN_CR3)
7580	{
7581	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7582	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7583	\|\| ( pVM->hm.s.fNestedPaging
7584	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7585	{
7586	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7587	VMXLOCAL_BREAK_RC(rc);
7588	if (pCtx->cr3 != u64Val)
7589	{
7590	pCtx->cr3 = u64Val;
7591	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7592	}
7593
7594	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7595	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7596	if (CPUMIsGuestInPAEModeEx(pCtx))
7597	{
7598	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7599	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7600	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7601	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7602	VMXLOCAL_BREAK_RC(rc);
7603	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7604	}
7605	}
7606	}
7607
7608	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7609	# if 0
7610	/** @todo NSTVMX: We handle each of these fields individually by passing it to IEM
7611	* VM-exit handlers. We might handle it differently when using the fast path. */
7612	/*
7613	* The hardware virtualization state currently consists of VMCS fields that may be
7614	* modified by execution of the nested-guest (that are not part of the general
7615	* guest state) and is visible to guest software. Hence, it is technically part of
7616	* the guest-CPU state when executing a nested-guest.
7617	*/
7618	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7619	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7620	{
7621	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7622	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7623	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7624	VMXLOCAL_BREAK_RC(rc);
7625
7626	/*
7627	* VM-entry can fail due to invalid-guest state, machine-check events and
7628	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7629	* all other VMCS fields are left unmodified on VM-entry failure.
7630	*
7631	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7632	*/
7633	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7634	if (!fEntryFailed)
7635	{
7636	/*
7637	* Some notes on VMCS fields that may need importing when the fast path
7638	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7639	*
7640	* Requires fixing up when using hardware-assisted VMX:
7641	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7642	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7643	* - IDT-vectoring info: Think about this.
7644	* - IDT-vectoring error code: Think about this.
7645	*
7646	* Emulated:
7647	* - Guest-interruptiblity state: Derived from FFs and RIP.
7648	* - Guest pending debug exceptions: Derived from DR6.
7649	* - Guest activity state: Emulated from EM state.
7650	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7651	* - Entry-interrupt info: Emulated, cleared to 0.
7652	*/
7653	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7654	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7655	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7656	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7657	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7658	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7659	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7660	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7661	/** @todo NSTVMX: Save and adjust preemption timer value. */
7662	}
7663
7664	VMXLOCAL_BREAK_RC(rc);
7665	}
7666	# endif
7667	#endif
7668	}
7669	} while (0);
7670
7671	if (RT_SUCCESS(rc))
7672	{
7673	/* Update fExtrn. */
7674	pCtx->fExtrn &= ~fWhat;
7675
7676	/* If everything has been imported, clear the HM keeper bit. */
7677	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7678	{
7679	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7680	Assert(!pCtx->fExtrn);
7681	}
7682	}
7683	}
7684	else
7685	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7686
7687	ASMSetFlags(fEFlags);
7688
7689	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7690
7691	if (RT_SUCCESS(rc))
7692	{ /* likely */ }
7693	else
7694	return rc;
7695
7696	/*
7697	* Honor any pending CR3 updates.
7698	*
7699	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7700	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7701	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7702	*
7703	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7704	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7705	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7706	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7707	*
7708	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7709	*/
7710	if (VMMRZCallRing3IsEnabled(pVCpu))
7711	{
7712	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7713	{
7714	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7715	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7716	}
7717
7718	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7719	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7720
7721	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7722	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7723	}
7724
7725	return VINF_SUCCESS;
7726	#undef VMXLOCAL_BREAK_RC
7727	}
7728
7729
7730	/**
7731	* Saves the guest state from the VMCS into the guest-CPU context.
7732	*
7733	* @returns VBox status code.
7734	* @param pVCpu The cross context virtual CPU structure.
7735	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7736	*/
7737	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7738	{
7739	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7740	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7741	}
7742
7743
7744	/**
7745	* Check per-VM and per-VCPU force flag actions that require us to go back to
7746	* ring-3 for one reason or another.
7747	*
7748	* @returns Strict VBox status code (i.e. informational status codes too)
7749	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7750	* ring-3.
7751	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7752	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7753	* interrupts)
7754	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7755	* all EMTs to be in ring-3.
7756	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7757	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7758	* to the EM loop.
7759	*
7760	* @param pVCpu The cross context virtual CPU structure.
7761	* @param fStepping Whether we are single-stepping the guest using the
7762	* hypervisor debugger.
7763	*/
7764	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7765	{
7766	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7767
7768	/*
7769	* Update pending interrupts into the APIC's IRR.
7770	*/
7771	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7772	APICUpdatePendingInterrupts(pVCpu);
7773
7774	/*
7775	* Anything pending? Should be more likely than not if we're doing a good job.
7776	*/
7777	PVM pVM = pVCpu->CTX_SUFF(pVM);
7778	if ( !fStepping
7779	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7780	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7781	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7782	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7783	return VINF_SUCCESS;
7784
7785	/* Pending PGM C3 sync. */
7786	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7787	{
7788	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7789	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7790	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7791	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7792	if (rcStrict2 != VINF_SUCCESS)
7793	{
7794	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7795	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7796	return rcStrict2;
7797	}
7798	}
7799
7800	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7801	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7802	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7803	{
7804	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7805	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7806	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7807	return rc2;
7808	}
7809
7810	/* Pending VM request packets, such as hardware interrupts. */
7811	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7812	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7813	{
7814	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7815	return VINF_EM_PENDING_REQUEST;
7816	}
7817
7818	/* Pending PGM pool flushes. */
7819	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7820	{
7821	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7822	return VINF_PGM_POOL_FLUSH_PENDING;
7823	}
7824
7825	/* Pending DMA requests. */
7826	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7827	{
7828	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7829	return VINF_EM_RAW_TO_R3;
7830	}
7831
7832	return VINF_SUCCESS;
7833	}
7834
7835
7836	/**
7837	* Converts any TRPM trap into a pending HM event. This is typically used when
7838	* entering from ring-3 (not longjmp returns).
7839	*
7840	* @param pVCpu The cross context virtual CPU structure.
7841	*/
7842	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7843	{
7844	Assert(TRPMHasTrap(pVCpu));
7845	Assert(!pVCpu->hm.s.Event.fPending);
7846
7847	uint8_t uVector;
7848	TRPMEVENT enmTrpmEvent;
7849	RTGCUINT uErrCode;
7850	RTGCUINTPTR GCPtrFaultAddress;
7851	uint8_t cbInstr;
7852
7853	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7854	AssertRC(rc);
7855
7856	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7857	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7858	if (enmTrpmEvent == TRPM_TRAP)
7859	{
7860	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7861	* generated using INT1 (ICEBP). */
7862	switch (uVector)
7863	{
7864	case X86_XCPT_NMI:
7865	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7866	break;
7867
7868	case X86_XCPT_BP:
7869	case X86_XCPT_OF:
7870	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7871	break;
7872
7873	case X86_XCPT_PF:
7874	case X86_XCPT_DF:
7875	case X86_XCPT_TS:
7876	case X86_XCPT_NP:
7877	case X86_XCPT_SS:
7878	case X86_XCPT_GP:
7879	case X86_XCPT_AC:
7880	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7881	RT_FALL_THRU();
7882	default:
7883	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7884	break;
7885	}
7886	}
7887	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7888	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7889	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7890	{
7891	switch (uVector)
7892	{
7893	case X86_XCPT_BP:
7894	case X86_XCPT_OF:
7895	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7896	break;
7897
7898	default:
7899	Assert(uVector == X86_XCPT_DB);
7900	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7901	break;
7902	}
7903	}
7904	else
7905	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7906
7907	rc = TRPMResetTrap(pVCpu);
7908	AssertRC(rc);
7909	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7910	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7911
7912	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7913	}
7914
7915
7916	/**
7917	* Converts the pending HM event into a TRPM trap.
7918	*
7919	* @param pVCpu The cross context virtual CPU structure.
7920	*/
7921	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7922	{
7923	Assert(pVCpu->hm.s.Event.fPending);
7924
7925	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7926	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7927	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7928	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7929
7930	/* If a trap was already pending, we did something wrong! */
7931	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7932
7933	/** @todo Use HMVmxEventToTrpmEventType() later. */
7934	TRPMEVENT enmTrapType;
7935	switch (uVectorType)
7936	{
7937	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7938	enmTrapType = TRPM_HARDWARE_INT;
7939	break;
7940
7941	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7942	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7943	enmTrapType = TRPM_TRAP;
7944	break;
7945
7946	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7947	Assert(uVector == X86_XCPT_DB);
7948	enmTrapType = TRPM_SOFTWARE_INT;
7949	break;
7950
7951	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7952	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7953	enmTrapType = TRPM_SOFTWARE_INT;
7954	break;
7955
7956	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7957	enmTrapType = TRPM_SOFTWARE_INT;
7958	break;
7959
7960	default:
7961	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7962	enmTrapType = TRPM_32BIT_HACK;
7963	break;
7964	}
7965
7966	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7967
7968	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7969	AssertRC(rc);
7970
7971	if (fErrorCodeValid)
7972	TRPMSetErrorCode(pVCpu, uErrorCode);
7973
7974	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7975	&& uVector == X86_XCPT_PF)
7976	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7977	else if (enmTrapType == TRPM_SOFTWARE_INT)
7978	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7979
7980	/* We're now done converting the pending event. */
7981	pVCpu->hm.s.Event.fPending = false;
7982	}
7983
7984
7985	/**
7986	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7987	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7988	*
7989	* @param pVCpu The cross context virtual CPU structure.
7990	* @param pVmcsInfo The VMCS info. object.
7991	*/
7992	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7993	{
7994	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7995	{
7996	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7997	{
7998	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7999	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8000	AssertRC(rc);
8001	}
8002	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8003	}
8004
8005
8006	/**
8007	* Clears the interrupt-window exiting control in the VMCS.
8008	*
8009	* @param pVmcsInfo The VMCS info. object.
8010	*/
8011	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8012	{
8013	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8014	{
8015	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8016	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8017	}
8018	return VINF_SUCCESS;
8019	}
8020
8021
8022	/**
8023	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8024	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8025	*
8026	* @param pVCpu The cross context virtual CPU structure.
8027	* @param pVmcsInfo The VMCS info. object.
8028	*/
8029	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8030	{
8031	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8032	{
8033	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8034	{
8035	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8036	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8037	AssertRC(rc);
8038	Log4Func(("Setup NMI-window exiting\n"));
8039	}
8040	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8041	}
8042
8043
8044	/**
8045	* Clears the NMI-window exiting control in the VMCS.
8046	*
8047	* @param pVmcsInfo The VMCS info. object.
8048	*/
8049	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8050	{
8051	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8052	{
8053	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8054	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8055	}
8056	return VINF_SUCCESS;
8057	}
8058
8059
8060	/**
8061	* Does the necessary state syncing before returning to ring-3 for any reason
8062	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8063	*
8064	* @returns VBox status code.
8065	* @param pVCpu The cross context virtual CPU structure.
8066	* @param fImportState Whether to import the guest state from the VMCS back
8067	* to the guest-CPU context.
8068	*
8069	* @remarks No-long-jmp zone!!!
8070	*/
8071	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8072	{
8073	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8074	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8075
8076	RTCPUID idCpu = RTMpCpuId();
8077	Log4Func(("HostCpuId=%u\n", idCpu));
8078
8079	/*
8080	* !!! IMPORTANT !!!
8081	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8082	*/
8083
8084	/* Save the guest state if necessary. */
8085	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8086	if (fImportState)
8087	{
8088	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8089	AssertRCReturn(rc, rc);
8090	}
8091
8092	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8093	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8094	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8095
8096	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8097	#ifdef VBOX_STRICT
8098	if (CPUMIsHyperDebugStateActive(pVCpu))
8099	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8100	#endif
8101	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8102	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8103	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8104
8105	#if HC_ARCH_BITS == 64
8106	/* Restore host-state bits that VT-x only restores partially. */
8107	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8108	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8109	{
8110	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8111	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8112	}
8113	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8114	#endif
8115
8116	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8117	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8118	{
8119	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8120	if (!fImportState)
8121	{
8122	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8123	AssertRCReturn(rc, rc);
8124	}
8125	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8126	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8127	}
8128	else
8129	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8130
8131	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8132	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8133
8134	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8135	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8136	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8137	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8138	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8139	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8140	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8141	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8142	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8143
8144	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8145
8146	/** @todo This partially defeats the purpose of having preemption hooks.
8147	* The problem is, deregistering the hooks should be moved to a place that
8148	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8149	* context.
8150	*/
8151	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8152	AssertRCReturn(rc, rc);
8153
8154	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8155	NOREF(idCpu);
8156	return VINF_SUCCESS;
8157	}
8158
8159
8160	/**
8161	* Leaves the VT-x session.
8162	*
8163	* @returns VBox status code.
8164	* @param pVCpu The cross context virtual CPU structure.
8165	*
8166	* @remarks No-long-jmp zone!!!
8167	*/
8168	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8169	{
8170	HM_DISABLE_PREEMPT(pVCpu);
8171	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8172	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8173	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8174
8175	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8176	and done this from the VMXR0ThreadCtxCallback(). */
8177	if (!pVCpu->hm.s.fLeaveDone)
8178	{
8179	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8180	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8181	pVCpu->hm.s.fLeaveDone = true;
8182	}
8183	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8184
8185	/*
8186	* !!! IMPORTANT !!!
8187	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8188	*/
8189
8190	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8191	/** @todo Deregistering here means we need to VMCLEAR always
8192	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8193	* for calling VMMR0ThreadCtxHookDisable here! */
8194	VMMR0ThreadCtxHookDisable(pVCpu);
8195
8196	/* Leave HM context. This takes care of local init (term). */
8197	int rc = HMR0LeaveCpu(pVCpu);
8198
8199	HM_RESTORE_PREEMPT();
8200	return rc;
8201	}
8202
8203
8204	/**
8205	* Does the necessary state syncing before doing a longjmp to ring-3.
8206	*
8207	* @returns VBox status code.
8208	* @param pVCpu The cross context virtual CPU structure.
8209	*
8210	* @remarks No-long-jmp zone!!!
8211	*/
8212	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8213	{
8214	return hmR0VmxLeaveSession(pVCpu);
8215	}
8216
8217
8218	/**
8219	* Take necessary actions before going back to ring-3.
8220	*
8221	* An action requires us to go back to ring-3. This function does the necessary
8222	* steps before we can safely return to ring-3. This is not the same as longjmps
8223	* to ring-3, this is voluntary and prepares the guest so it may continue
8224	* executing outside HM (recompiler/IEM).
8225	*
8226	* @returns VBox status code.
8227	* @param pVCpu The cross context virtual CPU structure.
8228	* @param rcExit The reason for exiting to ring-3. Can be
8229	* VINF_VMM_UNKNOWN_RING3_CALL.
8230	*/
8231	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8232	{
8233	Assert(pVCpu);
8234	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8235
8236	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8237	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8238	{
8239	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8240	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8241	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8242	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8243	}
8244
8245	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8246	VMMRZCallRing3Disable(pVCpu);
8247	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8248
8249	/*
8250	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8251	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8252	*
8253	* This is because execution may continue from ring-3 and we would need to inject
8254	* the event from there (hence place it back in TRPM).
8255	*/
8256	if (pVCpu->hm.s.Event.fPending)
8257	{
8258	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8259	Assert(!pVCpu->hm.s.Event.fPending);
8260
8261	/* Clear the events from the VMCS. */
8262	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8263	AssertRCReturn(rc, rc);
8264	}
8265	#ifdef VBOX_STRICT
8266	else
8267	{
8268	/*
8269	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8270	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8271	* occasionally, see @bugref{9180#c42}.
8272	*/
8273	uint32_t uEntryIntInfo;
8274	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8275	AssertRC(rc);
8276	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8277	}
8278	#endif
8279
8280	/*
8281	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8282	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8283	* (e.g. TPR below threshold).
8284	*/
8285	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8286	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8287	AssertRCReturn(rc, rc);
8288
8289	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8290	and if we're injecting an event we should have a TRPM trap pending. */
8291	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8292	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8293	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8294	#endif
8295
8296	/* Save guest state and restore host state bits. */
8297	rc = hmR0VmxLeaveSession(pVCpu);
8298	AssertRCReturn(rc, rc);
8299	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8300
8301	/* Thread-context hooks are unregistered at this point!!! */
8302
8303	/* Sync recompiler state. */
8304	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8305	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8306	\| CPUM_CHANGED_LDTR
8307	\| CPUM_CHANGED_GDTR
8308	\| CPUM_CHANGED_IDTR
8309	\| CPUM_CHANGED_TR
8310	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8311	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8312	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8313	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8314
8315	Assert(!pVCpu->hm.s.fClearTrapFlag);
8316
8317	/* Update the exit-to-ring 3 reason. */
8318	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8319
8320	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8321	if ( rcExit != VINF_EM_RAW_INTERRUPT
8322	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8323	{
8324	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8325	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8326	}
8327
8328	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8329
8330	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8331	VMMRZCallRing3RemoveNotification(pVCpu);
8332	VMMRZCallRing3Enable(pVCpu);
8333
8334	return rc;
8335	}
8336
8337
8338	/**
8339	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8340	* longjump to ring-3 and possibly get preempted.
8341	*
8342	* @returns VBox status code.
8343	* @param pVCpu The cross context virtual CPU structure.
8344	* @param enmOperation The operation causing the ring-3 longjump.
8345	* @param pvUser User argument, currently unused, NULL.
8346	*/
8347	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8348	{
8349	RT_NOREF(pvUser);
8350	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8351	{
8352	/*
8353	* !!! IMPORTANT !!!
8354	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8355	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8356	*/
8357	VMMRZCallRing3RemoveNotification(pVCpu);
8358	VMMRZCallRing3Disable(pVCpu);
8359	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8360	RTThreadPreemptDisable(&PreemptState);
8361
8362	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8363	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8364	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8365	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8366
8367	#if HC_ARCH_BITS == 64
8368	/* Restore host-state bits that VT-x only restores partially. */
8369	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8370	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8371	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8372	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8373	#endif
8374
8375	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8376	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8377	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8378
8379	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8380	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8381	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8382
8383	/* Clear the current VMCS data back to memory. */
8384	hmR0VmxClearVmcs(pVmcsInfo);
8385
8386	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8387	VMMR0ThreadCtxHookDisable(pVCpu);
8388	HMR0LeaveCpu(pVCpu);
8389	RTThreadPreemptRestore(&PreemptState);
8390	return VINF_SUCCESS;
8391	}
8392
8393	Assert(pVCpu);
8394	Assert(pvUser);
8395	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8396	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8397
8398	VMMRZCallRing3Disable(pVCpu);
8399	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8400
8401	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8402
8403	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8404	AssertRCReturn(rc, rc);
8405
8406	VMMRZCallRing3Enable(pVCpu);
8407	return VINF_SUCCESS;
8408	}
8409
8410
8411	/**
8412	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8413	* stack.
8414	*
8415	* @returns Strict VBox status code (i.e. informational status codes too).
8416	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8417	* @param pVCpu The cross context virtual CPU structure.
8418	* @param uValue The value to push to the guest stack.
8419	*/
8420	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8421	{
8422	/*
8423	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8424	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8425	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8426	*/
8427	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8428	if (pCtx->sp == 1)
8429	return VINF_EM_RESET;
8430	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8431	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8432	AssertRC(rc);
8433	return rc;
8434	}
8435
8436
8437	/**
8438	* Injects an event into the guest upon VM-entry by updating the relevant fields
8439	* in the VM-entry area in the VMCS.
8440	*
8441	* @returns Strict VBox status code (i.e. informational status codes too).
8442	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8443	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8444	*
8445	* @param pVCpu The cross context virtual CPU structure.
8446	* @param pVmxTransient The VMX-transient structure.
8447	* @param pEvent The event being injected.
8448	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8449	* This will be updated if necessary. This cannot not
8450	* be NULL.
8451	* @param fStepping Whether we're single-stepping guest execution and
8452	* should return VINF_EM_DBG_STEPPED if the event is
8453	* injected directly (registers modified by us, not by
8454	* hardware on VM-entry).
8455	*/
8456	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8457	uint32_t *pfIntrState)
8458	{
8459	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8460	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8461	Assert(pfIntrState);
8462
8463	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8464	uint32_t u32IntInfo = pEvent->u64IntInfo;
8465	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8466	uint32_t const cbInstr = pEvent->cbInstr;
8467	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8468	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8469	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8470
8471	#ifdef VBOX_STRICT
8472	/*
8473	* Validate the error-code-valid bit for hardware exceptions.
8474	* No error codes for exceptions in real-mode.
8475	*
8476	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8477	*/
8478	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8479	&& !CPUMIsGuestInRealModeEx(pCtx))
8480	{
8481	switch (uVector)
8482	{
8483	case X86_XCPT_PF:
8484	case X86_XCPT_DF:
8485	case X86_XCPT_TS:
8486	case X86_XCPT_NP:
8487	case X86_XCPT_SS:
8488	case X86_XCPT_GP:
8489	case X86_XCPT_AC:
8490	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8491	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8492	RT_FALL_THRU();
8493	default:
8494	break;
8495	}
8496	}
8497
8498	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8499	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8500	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8501	#endif
8502
8503	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8504
8505	/*
8506	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8507	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8508	* interrupt handler in the (real-mode) guest.
8509	*
8510	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8511	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8512	*/
8513	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8514	{
8515	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8516	{
8517	/*
8518	* For CPUs with unrestricted guest execution enabled and with the guest
8519	* in real-mode, we must not set the deliver-error-code bit.
8520	*
8521	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8522	*/
8523	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8524	}
8525	else
8526	{
8527	PVM pVM = pVCpu->CTX_SUFF(pVM);
8528	Assert(PDMVmmDevHeapIsEnabled(pVM));
8529	Assert(pVM->hm.s.vmx.pRealModeTSS);
8530	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8531
8532	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8533	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8534	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8535	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8536	AssertRCReturn(rc2, rc2);
8537
8538	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8539	size_t const cbIdtEntry = sizeof(X86IDTR16);
8540	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8541	{
8542	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8543	if (uVector == X86_XCPT_DF)
8544	return VINF_EM_RESET;
8545
8546	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8547	No error codes for exceptions in real-mode. */
8548	if (uVector == X86_XCPT_GP)
8549	{
8550	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8551	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8552	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8553	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8554	HMEVENT EventXcptDf;
8555	RT_ZERO(EventXcptDf);
8556	EventXcptDf.u64IntInfo = uXcptDfInfo;
8557	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8558	}
8559
8560	/*
8561	* If we're injecting an event with no valid IDT entry, inject a #GP.
8562	* No error codes for exceptions in real-mode.
8563	*
8564	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8565	*/
8566	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8567	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8568	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8569	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8570	HMEVENT EventXcptGp;
8571	RT_ZERO(EventXcptGp);
8572	EventXcptGp.u64IntInfo = uXcptGpInfo;
8573	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8574	}
8575
8576	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8577	uint16_t uGuestIp = pCtx->ip;
8578	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8579	{
8580	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8581	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8582	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8583	}
8584	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8585	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8586
8587	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8588	X86IDTR16 IdtEntry;
8589	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8590	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8591	AssertRCReturn(rc2, rc2);
8592
8593	/* Construct the stack frame for the interrupt/exception handler. */
8594	VBOXSTRICTRC rcStrict;
8595	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8596	if (rcStrict == VINF_SUCCESS)
8597	{
8598	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8599	if (rcStrict == VINF_SUCCESS)
8600	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8601	}
8602
8603	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8604	if (rcStrict == VINF_SUCCESS)
8605	{
8606	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8607	pCtx->rip = IdtEntry.offSel;
8608	pCtx->cs.Sel = IdtEntry.uSel;
8609	pCtx->cs.ValidSel = IdtEntry.uSel;
8610	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8611	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8612	&& uVector == X86_XCPT_PF)
8613	pCtx->cr2 = GCPtrFault;
8614
8615	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8616	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8617	\| HM_CHANGED_GUEST_RSP);
8618
8619	/*
8620	* If we delivered a hardware exception (other than an NMI) and if there was
8621	* block-by-STI in effect, we should clear it.
8622	*/
8623	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8624	{
8625	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8626	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8627	Log4Func(("Clearing inhibition due to STI\n"));
8628	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8629	}
8630
8631	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8632	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8633
8634	/*
8635	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8636	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8637	*/
8638	pVCpu->hm.s.Event.fPending = false;
8639
8640	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8641	if (fStepping)
8642	rcStrict = VINF_EM_DBG_STEPPED;
8643	}
8644	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8645	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8646	return rcStrict;
8647	}
8648	}
8649
8650	/*
8651	* Validate.
8652	*/
8653	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8654	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8655
8656	/*
8657	* Inject the event into the VMCS.
8658	*/
8659	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8660	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8661	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8662	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8663	AssertRCReturn(rc, rc);
8664
8665	/*
8666	* Update guest CR2 if this is a page-fault.
8667	*/
8668	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8669	&& uVector == X86_XCPT_PF)
8670	pCtx->cr2 = GCPtrFault;
8671
8672	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8673	return VINF_SUCCESS;
8674	}
8675
8676
8677	/**
8678	* Evaluates the event to be delivered to the guest and sets it as the pending
8679	* event.
8680	*
8681	* @returns Strict VBox status code (i.e. informational status codes too).
8682	* @param pVCpu The cross context virtual CPU structure.
8683	* @param pVmxTransient The VMX-transient structure.
8684	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8685	*/
8686	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8687	{
8688	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8690
8691	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8692	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8693	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8694	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8695	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8696
8697	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8698	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8699	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8700	Assert(!TRPMHasTrap(pVCpu));
8701	Assert(pfIntrState);
8702
8703	*pfIntrState = fIntrState;
8704
8705	/*
8706	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8707	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8708	*/
8709	/** @todo SMI. SMIs take priority over NMIs. */
8710	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8711	{
8712	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8713	if ( !pVCpu->hm.s.Event.fPending
8714	&& !fBlockNmi
8715	&& !fBlockSti
8716	&& !fBlockMovSS)
8717	{
8718	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8719	if ( pVmxTransient->fIsNestedGuest
8720	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8721	return IEMExecVmxVmexitNmi(pVCpu);
8722	#endif
8723	hmR0VmxSetPendingXcptNmi(pVCpu);
8724	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8725	Log4Func(("Pending NMI\n"));
8726	}
8727	else
8728	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8729	}
8730	/*
8731	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8732	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8733	*/
8734	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8735	&& !pVCpu->hm.s.fSingleInstruction)
8736	{
8737	Assert(!DBGFIsStepping(pVCpu));
8738	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8739	AssertRCReturn(rc, rc);
8740	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8741	if ( !pVCpu->hm.s.Event.fPending
8742	&& !fBlockInt
8743	&& !fBlockSti
8744	&& !fBlockMovSS)
8745	{
8746	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8747	if ( pVmxTransient->fIsNestedGuest
8748	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8749	{
8750	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8751	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8752	return rcStrict;
8753	}
8754	#endif
8755	uint8_t u8Interrupt;
8756	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8757	if (RT_SUCCESS(rc))
8758	{
8759	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8760	if ( pVmxTransient->fIsNestedGuest
8761	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8762	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8763	{
8764	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8765	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8766	return rcStrict;
8767	}
8768	#endif
8769	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8770	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8771	}
8772	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8773	{
8774	if ( !pVmxTransient->fIsNestedGuest
8775	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8776	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8777	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8778
8779	/*
8780	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8781	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8782	* need to re-set this force-flag here.
8783	*/
8784	}
8785	else
8786	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8787	}
8788	else
8789	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8790	}
8791
8792	return VINF_SUCCESS;
8793	}
8794
8795
8796	/**
8797	* Injects any pending events into the guest if the guest is in a state to
8798	* receive them.
8799	*
8800	* @returns Strict VBox status code (i.e. informational status codes too).
8801	* @param pVCpu The cross context virtual CPU structure.
8802	* @param pVmxTransient The VMX-transient structure.
8803	* @param fIntrState The VT-x guest-interruptibility state.
8804	* @param fStepping Whether we are single-stepping the guest using the
8805	* hypervisor debugger and should return
8806	* VINF_EM_DBG_STEPPED if the event was dispatched
8807	* directly.
8808	*/
8809	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8810	{
8811	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8812	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8813
8814	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8815	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8816
8817	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8818	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8819	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8820	Assert(!TRPMHasTrap(pVCpu));
8821
8822	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8823	if (pVCpu->hm.s.Event.fPending)
8824	{
8825	/*
8826	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8827	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8828	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8829	*
8830	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8831	*/
8832	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8833	#ifdef VBOX_STRICT
8834	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8835	{
8836	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8837	Assert(!fBlockInt);
8838	Assert(!fBlockSti);
8839	Assert(!fBlockMovSS);
8840	}
8841	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8842	{
8843	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8844	Assert(!fBlockSti);
8845	Assert(!fBlockMovSS);
8846	Assert(!fBlockNmi);
8847	}
8848	#endif
8849	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8850	uIntType));
8851
8852	/*
8853	* Inject the event and get any changes to the guest-interruptibility state.
8854	*
8855	* The guest-interruptibility state may need to be updated if we inject the event
8856	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8857	*/
8858	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8859	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8860
8861	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8862	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8863	else
8864	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8865	}
8866
8867	/*
8868	* Update the guest-interruptibility state.
8869	*
8870	* This is required for the real-on-v86 software interrupt injection case above, as well as
8871	* updates to the guest state from ring-3 or IEM/REM.
8872	*/
8873	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8874	AssertRCReturn(rc, rc);
8875
8876	/*
8877	* There's no need to clear the VM-entry interruption-information field here if we're not
8878	* injecting anything. VT-x clears the valid bit on every VM-exit.
8879	*
8880	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8881	*/
8882
8883	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8884	NOREF(fBlockMovSS); NOREF(fBlockSti);
8885	return rcStrict;
8886	}
8887
8888
8889	/**
8890	* Enters the VT-x session.
8891	*
8892	* @returns VBox status code.
8893	* @param pVCpu The cross context virtual CPU structure.
8894	*/
8895	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8896	{
8897	AssertPtr(pVCpu);
8898	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8900
8901	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8902	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8903	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8904
8905	#ifdef VBOX_STRICT
8906	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8907	RTCCUINTREG uHostCR4 = ASMGetCR4();
8908	if (!(uHostCR4 & X86_CR4_VMXE))
8909	{
8910	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8911	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8912	}
8913	#endif
8914
8915	/*
8916	* Load the appropriate VMCS as the current and active one.
8917	*/
8918	PVMXVMCSINFO pVmcsInfo;
8919	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8920	if (!fInNestedGuestMode)
8921	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8922	else
8923	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8924	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8925	if (RT_SUCCESS(rc))
8926	{
8927	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8928	pVCpu->hm.s.fLeaveDone = false;
8929	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8930
8931	/*
8932	* Do the EMT scheduled L1D flush here if needed.
8933	*/
8934	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8935	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8936	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8937	hmR0MdsClear();
8938	}
8939	return rc;
8940	}
8941
8942
8943	/**
8944	* The thread-context callback (only on platforms which support it).
8945	*
8946	* @param enmEvent The thread-context event.
8947	* @param pVCpu The cross context virtual CPU structure.
8948	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8949	* @thread EMT(pVCpu)
8950	*/
8951	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8952	{
8953	NOREF(fGlobalInit);
8954
8955	switch (enmEvent)
8956	{
8957	case RTTHREADCTXEVENT_OUT:
8958	{
8959	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8960	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8961	VMCPU_ASSERT_EMT(pVCpu);
8962
8963	/* No longjmps (logger flushes, locks) in this fragile context. */
8964	VMMRZCallRing3Disable(pVCpu);
8965	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8966
8967	/* Restore host-state (FPU, debug etc.) */
8968	if (!pVCpu->hm.s.fLeaveDone)
8969	{
8970	/*
8971	* Do -not- import the guest-state here as we might already be in the middle of importing
8972	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8973	*/
8974	hmR0VmxLeave(pVCpu, false /* fImportState */);
8975	pVCpu->hm.s.fLeaveDone = true;
8976	}
8977
8978	/* Leave HM context, takes care of local init (term). */
8979	int rc = HMR0LeaveCpu(pVCpu);
8980	AssertRC(rc);
8981
8982	/* Restore longjmp state. */
8983	VMMRZCallRing3Enable(pVCpu);
8984	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8985	break;
8986	}
8987
8988	case RTTHREADCTXEVENT_IN:
8989	{
8990	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8991	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8992	VMCPU_ASSERT_EMT(pVCpu);
8993
8994	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8995	VMMRZCallRing3Disable(pVCpu);
8996	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8997
8998	/* Initialize the bare minimum state required for HM. This takes care of
8999	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9000	int rc = hmR0EnterCpu(pVCpu);
9001	AssertRC(rc);
9002	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9003	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9004
9005	/* Load the active VMCS as the current one. */
9006	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9007	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9008	AssertRC(rc);
9009	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9010	pVCpu->hm.s.fLeaveDone = false;
9011
9012	/* Do the EMT scheduled L1D flush if needed. */
9013	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9014	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9015
9016	/* Restore longjmp state. */
9017	VMMRZCallRing3Enable(pVCpu);
9018	break;
9019	}
9020
9021	default:
9022	break;
9023	}
9024	}
9025
9026
9027	/**
9028	* Exports the host state into the VMCS host-state area.
9029	* Sets up the VM-exit MSR-load area.
9030	*
9031	* The CPU state will be loaded from these fields on every successful VM-exit.
9032	*
9033	* @returns VBox status code.
9034	* @param pVCpu The cross context virtual CPU structure.
9035	*
9036	* @remarks No-long-jump zone!!!
9037	*/
9038	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9039	{
9040	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9041
9042	int rc = VINF_SUCCESS;
9043	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9044	{
9045	rc = hmR0VmxExportHostControlRegs();
9046	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9047
9048	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9049	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9050
9051	rc = hmR0VmxExportHostMsrs(pVCpu);
9052	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9053
9054	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9055	}
9056	return rc;
9057	}
9058
9059
9060	/**
9061	* Saves the host state in the VMCS host-state.
9062	*
9063	* @returns VBox status code.
9064	* @param pVCpu The cross context virtual CPU structure.
9065	*
9066	* @remarks No-long-jump zone!!!
9067	*/
9068	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9069	{
9070	AssertPtr(pVCpu);
9071	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9072
9073	/*
9074	* Export the host state here while entering HM context.
9075	* When thread-context hooks are used, we might get preempted and have to re-save the host
9076	* state but most of the time we won't be, so do it here before we disable interrupts.
9077	*/
9078	return hmR0VmxExportHostState(pVCpu);
9079	}
9080
9081
9082	/**
9083	* Exports the guest state into the VMCS guest-state area.
9084	*
9085	* The will typically be done before VM-entry when the guest-CPU state and the
9086	* VMCS state may potentially be out of sync.
9087	*
9088	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9089	* VM-entry controls.
9090	* Sets up the appropriate VMX non-root function to execute guest code based on
9091	* the guest CPU mode.
9092	*
9093	* @returns VBox strict status code.
9094	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9095	* without unrestricted guest execution and the VMMDev is not presently
9096	* mapped (e.g. EFI32).
9097	*
9098	* @param pVCpu The cross context virtual CPU structure.
9099	* @param pVmxTransient The VMX-transient structure.
9100	*
9101	* @remarks No-long-jump zone!!!
9102	*/
9103	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9104	{
9105	AssertPtr(pVCpu);
9106	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9107	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9108
9109	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9110
9111	/*
9112	* Determine real-on-v86 mode.
9113	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9114	*/
9115	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9116	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9117	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9118	pVmcsInfo->RealMode. fRealOnV86Active = false;
9119	else
9120	{
9121	Assert(!pVmxTransient->fIsNestedGuest);
9122	pVmcsInfo->RealMode.fRealOnV86Active = true;
9123	}
9124
9125	/*
9126	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9127	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9128	*/
9129	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9130	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9131	* be a need to evaluate this everytime since I'm pretty sure we intercept
9132	* all guest paging mode changes. */
9133	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9134	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9135
9136	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9137	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9138
9139	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9140	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9141
9142	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9143	if (rcStrict == VINF_SUCCESS)
9144	{ /* likely */ }
9145	else
9146	{
9147	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9148	return rcStrict;
9149	}
9150
9151	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9152	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9153
9154	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9155	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9156
9157	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9158	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9159
9160	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9161	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9162
9163	rc = hmR0VmxExportGuestRip(pVCpu);
9164	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9165	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9166	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9167
9168	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9169	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9170	\| HM_CHANGED_GUEST_CR2
9171	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9172	\| HM_CHANGED_GUEST_X87
9173	\| HM_CHANGED_GUEST_SSE_AVX
9174	\| HM_CHANGED_GUEST_OTHER_XSAVE
9175	\| HM_CHANGED_GUEST_XCRx
9176	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9177	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9178	\| HM_CHANGED_GUEST_TSC_AUX
9179	\| HM_CHANGED_GUEST_OTHER_MSRS
9180	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9181	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9182
9183	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9184	return rc;
9185	}
9186
9187
9188	/**
9189	* Exports the state shared between the host and guest into the VMCS.
9190	*
9191	* @param pVCpu The cross context virtual CPU structure.
9192	* @param pVmxTransient The VMX-transient structure.
9193	*
9194	* @remarks No-long-jump zone!!!
9195	*/
9196	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9197	{
9198	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9199	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9200
9201	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9202	{
9203	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9204	AssertRC(rc);
9205	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9206
9207	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9208	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9209	{
9210	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9211	AssertRC(rc);
9212	}
9213	}
9214
9215	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9216	{
9217	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9218	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9219	}
9220
9221	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9222	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9223	}
9224
9225
9226	/**
9227	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9228	*
9229	* @returns Strict VBox status code (i.e. informational status codes too).
9230	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9231	* without unrestricted guest execution and the VMMDev is not presently
9232	* mapped (e.g. EFI32).
9233	*
9234	* @param pVCpu The cross context virtual CPU structure.
9235	* @param pVmxTransient The VMX-transient structure.
9236	*
9237	* @remarks No-long-jump zone!!!
9238	*/
9239	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9240	{
9241	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9242	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9243	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9244
9245	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9246	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9247	#endif
9248
9249	/*
9250	* For many exits it's only RIP that changes and hence try to export it first
9251	* without going through a lot of change flag checks.
9252	*/
9253	VBOXSTRICTRC rcStrict;
9254	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9255	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9256	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9257	{
9258	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9259	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9260	{ /* likely */}
9261	else
9262	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9263	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9264	}
9265	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9266	{
9267	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9268	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9269	{ /* likely */}
9270	else
9271	{
9272	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9273	VBOXSTRICTRC_VAL(rcStrict)));
9274	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9275	return rcStrict;
9276	}
9277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9278	}
9279	else
9280	rcStrict = VINF_SUCCESS;
9281
9282	#ifdef VBOX_STRICT
9283	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9284	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9285	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9286	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9287	("fCtxChanged=%#RX64\n", fCtxChanged));
9288	#endif
9289	return rcStrict;
9290	}
9291
9292
9293	/**
9294	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9295	* and update error record fields accordingly.
9296	*
9297	* @return VMX_IGS_* return codes.
9298	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9299	* wrong with the guest state.
9300	*
9301	* @param pVCpu The cross context virtual CPU structure.
9302	* @param pVmcsInfo The VMCS info. object.
9303	*
9304	* @remarks This function assumes our cache of the VMCS controls
9305	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9306	*/
9307	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9308	{
9309	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9310	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9311	uError = (err); \
9312	break; \
9313	} else do { } while (0)
9314
9315	int rc;
9316	PVM pVM = pVCpu->CTX_SUFF(pVM);
9317	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9318	uint32_t uError = VMX_IGS_ERROR;
9319	uint32_t u32Val;
9320	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9321
9322	do
9323	{
9324	/*
9325	* CR0.
9326	*/
9327	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9328	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9329	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9330	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9331	if (fUnrestrictedGuest)
9332	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9333
9334	uint32_t u32GuestCr0;
9335	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9336	AssertRCBreak(rc);
9337	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9338	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9339	if ( !fUnrestrictedGuest
9340	&& (u32GuestCr0 & X86_CR0_PG)
9341	&& !(u32GuestCr0 & X86_CR0_PE))
9342	{
9343	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9344	}
9345
9346	/*
9347	* CR4.
9348	*/
9349	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9350	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9351
9352	uint32_t u32GuestCr4;
9353	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9354	AssertRCBreak(rc);
9355	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9356	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9357
9358	/*
9359	* IA32_DEBUGCTL MSR.
9360	*/
9361	uint64_t u64Val;
9362	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9363	AssertRCBreak(rc);
9364	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9365	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9366	{
9367	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9368	}
9369	uint64_t u64DebugCtlMsr = u64Val;
9370
9371	#ifdef VBOX_STRICT
9372	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9373	AssertRCBreak(rc);
9374	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9375	#endif
9376	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9377
9378	/*
9379	* RIP and RFLAGS.
9380	*/
9381	uint32_t u32Eflags;
9382	#if HC_ARCH_BITS == 64
9383	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9384	AssertRCBreak(rc);
9385	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9386	if ( !fLongModeGuest
9387	\|\| !pCtx->cs.Attr.n.u1Long)
9388	{
9389	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9390	}
9391	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9392	* must be identical if the "IA-32e mode guest" VM-entry
9393	* control is 1 and CS.L is 1. No check applies if the
9394	* CPU supports 64 linear-address bits. */
9395
9396	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9397	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9398	AssertRCBreak(rc);
9399	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9400	VMX_IGS_RFLAGS_RESERVED);
9401	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9402	u32Eflags = u64Val;
9403	#else
9404	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9405	AssertRCBreak(rc);
9406	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9407	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9408	#endif
9409
9410	if ( fLongModeGuest
9411	\|\| ( fUnrestrictedGuest
9412	&& !(u32GuestCr0 & X86_CR0_PE)))
9413	{
9414	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9415	}
9416
9417	uint32_t u32EntryInfo;
9418	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9419	AssertRCBreak(rc);
9420	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9421	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9422	{
9423	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9424	}
9425
9426	/*
9427	* 64-bit checks.
9428	*/
9429	#if HC_ARCH_BITS == 64
9430	if (fLongModeGuest)
9431	{
9432	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9433	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9434	}
9435
9436	if ( !fLongModeGuest
9437	&& (u32GuestCr4 & X86_CR4_PCIDE))
9438	{
9439	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9440	}
9441
9442	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9443	* 51:32 beyond the processor's physical-address width are 0. */
9444
9445	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9446	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9447	{
9448	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9449	}
9450
9451	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9452	AssertRCBreak(rc);
9453	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9454
9455	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9456	AssertRCBreak(rc);
9457	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9458	#endif
9459
9460	/*
9461	* PERF_GLOBAL MSR.
9462	*/
9463	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9464	{
9465	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9466	AssertRCBreak(rc);
9467	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9468	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9469	}
9470
9471	/*
9472	* PAT MSR.
9473	*/
9474	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9475	{
9476	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9477	AssertRCBreak(rc);
9478	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9479	for (unsigned i = 0; i < 8; i++)
9480	{
9481	uint8_t u8Val = (u64Val & 0xff);
9482	if ( u8Val != 0 /* UC */
9483	&& u8Val != 1 /* WC */
9484	&& u8Val != 4 /* WT */
9485	&& u8Val != 5 /* WP */
9486	&& u8Val != 6 /* WB */
9487	&& u8Val != 7 /* UC- */)
9488	{
9489	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9490	}
9491	u64Val >>= 8;
9492	}
9493	}
9494
9495	/*
9496	* EFER MSR.
9497	*/
9498	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9499	{
9500	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9501	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9502	AssertRCBreak(rc);
9503	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9504	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9505	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9506	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9507	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9508	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9509	* iemVmxVmentryCheckGuestState(). */
9510	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9511	\|\| !(u32GuestCr0 & X86_CR0_PG)
9512	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9513	VMX_IGS_EFER_LMA_LME_MISMATCH);
9514	}
9515
9516	/*
9517	* Segment registers.
9518	*/
9519	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9520	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9521	if (!(u32Eflags & X86_EFL_VM))
9522	{
9523	/* CS */
9524	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9525	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9526	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9527	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9528	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9529	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9530	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9531	/* CS cannot be loaded with NULL in protected mode. */
9532	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9533	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9534	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9535	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9536	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9537	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9538	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9539	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9540	else
9541	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9542
9543	/* SS */
9544	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9545	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9546	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9547	if ( !(pCtx->cr0 & X86_CR0_PE)
9548	\|\| pCtx->cs.Attr.n.u4Type == 3)
9549	{
9550	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9551	}
9552	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9553	{
9554	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9555	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9556	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9557	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9558	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9559	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9560	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9561	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9562	}
9563
9564	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9565	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9566	{
9567	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9568	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9569	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9570	\|\| pCtx->ds.Attr.n.u4Type > 11
9571	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9572	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9573	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9574	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9575	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9576	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9577	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9578	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9579	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9580	}
9581	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9582	{
9583	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9584	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9585	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9586	\|\| pCtx->es.Attr.n.u4Type > 11
9587	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9588	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9589	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9590	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9591	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9592	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9593	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9594	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9595	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9596	}
9597	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9598	{
9599	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9600	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9601	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9602	\|\| pCtx->fs.Attr.n.u4Type > 11
9603	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9604	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9605	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9606	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9607	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9608	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9609	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9610	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9611	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9612	}
9613	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9614	{
9615	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9616	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9617	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9618	\|\| pCtx->gs.Attr.n.u4Type > 11
9619	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9620	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9621	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9622	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9623	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9624	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9625	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9626	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9627	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9628	}
9629	/* 64-bit capable CPUs. */
9630	#if HC_ARCH_BITS == 64
9631	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9632	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9633	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9634	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9635	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9636	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9637	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9638	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9639	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9640	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9641	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9642	#endif
9643	}
9644	else
9645	{
9646	/* V86 mode checks. */
9647	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9648	if (pVmcsInfo->RealMode.fRealOnV86Active)
9649	{
9650	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9651	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9652	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9653	}
9654	else
9655	{
9656	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9657	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9658	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9659	}
9660
9661	/* CS */
9662	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9663	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9664	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9665	/* SS */
9666	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9667	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9668	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9669	/* DS */
9670	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9671	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9672	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9673	/* ES */
9674	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9675	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9676	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9677	/* FS */
9678	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9679	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9680	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9681	/* GS */
9682	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9683	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9684	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9685	/* 64-bit capable CPUs. */
9686	#if HC_ARCH_BITS == 64
9687	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9688	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9689	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9690	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9691	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9692	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9693	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9694	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9695	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9696	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9697	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9698	#endif
9699	}
9700
9701	/*
9702	* TR.
9703	*/
9704	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9705	/* 64-bit capable CPUs. */
9706	#if HC_ARCH_BITS == 64
9707	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9708	#endif
9709	if (fLongModeGuest)
9710	{
9711	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9712	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9713	}
9714	else
9715	{
9716	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9717	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9718	VMX_IGS_TR_ATTR_TYPE_INVALID);
9719	}
9720	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9721	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9722	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9723	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9724	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9725	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9726	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9727	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9728
9729	/*
9730	* GDTR and IDTR.
9731	*/
9732	#if HC_ARCH_BITS == 64
9733	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9734	AssertRCBreak(rc);
9735	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9736
9737	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9738	AssertRCBreak(rc);
9739	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9740	#endif
9741
9742	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9743	AssertRCBreak(rc);
9744	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9745
9746	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9747	AssertRCBreak(rc);
9748	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9749
9750	/*
9751	* Guest Non-Register State.
9752	*/
9753	/* Activity State. */
9754	uint32_t u32ActivityState;
9755	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9756	AssertRCBreak(rc);
9757	HMVMX_CHECK_BREAK( !u32ActivityState
9758	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9759	VMX_IGS_ACTIVITY_STATE_INVALID);
9760	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9761	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9762	uint32_t u32IntrState;
9763	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9764	AssertRCBreak(rc);
9765	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9766	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9767	{
9768	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9769	}
9770
9771	/** @todo Activity state and injecting interrupts. Left as a todo since we
9772	* currently don't use activity states but ACTIVE. */
9773
9774	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9775	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9776
9777	/* Guest interruptibility-state. */
9778	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9779	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9780	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9781	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9782	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9783	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9784	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9785	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9786	{
9787	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9788	{
9789	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9790	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9791	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9792	}
9793	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9794	{
9795	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9796	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9797	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9798	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9799	}
9800	}
9801	/** @todo Assumes the processor is not in SMM. */
9802	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9803	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9804	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9805	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9806	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9807	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9808	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9809	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9810	{
9811	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9812	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9813	}
9814
9815	/* Pending debug exceptions. */
9816	#if HC_ARCH_BITS == 64
9817	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9818	AssertRCBreak(rc);
9819	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9820	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9821	u32Val = u64Val; /* For pending debug exceptions checks below. */
9822	#else
9823	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9824	AssertRCBreak(rc);
9825	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9826	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9827	#endif
9828
9829	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9830	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9831	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9832	{
9833	if ( (u32Eflags & X86_EFL_TF)
9834	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9835	{
9836	/* Bit 14 is PendingDebug.BS. */
9837	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9838	}
9839	if ( !(u32Eflags & X86_EFL_TF)
9840	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9841	{
9842	/* Bit 14 is PendingDebug.BS. */
9843	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9844	}
9845	}
9846
9847	/* VMCS link pointer. */
9848	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9849	AssertRCBreak(rc);
9850	if (u64Val != UINT64_C(0xffffffffffffffff))
9851	{
9852	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9853	/** @todo Bits beyond the processor's physical-address width MBZ. */
9854	/** @todo 32-bit located in memory referenced by value of this field (as a
9855	* physical address) must contain the processor's VMCS revision ID. */
9856	/** @todo SMM checks. */
9857	}
9858
9859	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9860	* not using nested paging? */
9861	if ( pVM->hm.s.fNestedPaging
9862	&& !fLongModeGuest
9863	&& CPUMIsGuestInPAEModeEx(pCtx))
9864	{
9865	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9866	AssertRCBreak(rc);
9867	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9868
9869	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9870	AssertRCBreak(rc);
9871	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9872
9873	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9874	AssertRCBreak(rc);
9875	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9876
9877	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9878	AssertRCBreak(rc);
9879	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9880	}
9881
9882	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9883	if (uError == VMX_IGS_ERROR)
9884	uError = VMX_IGS_REASON_NOT_FOUND;
9885	} while (0);
9886
9887	pVCpu->hm.s.u32HMError = uError;
9888	return uError;
9889
9890	#undef HMVMX_ERROR_BREAK
9891	#undef HMVMX_CHECK_BREAK
9892	}
9893
9894
9895	/**
9896	* Setup the APIC-access page for virtualizing APIC access.
9897	*
9898	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9899	* this not done as part of exporting guest state, see @bugref{8721}.
9900	*
9901	* @returns VBox status code.
9902	* @param pVCpu The cross context virtual CPU structure.
9903	*/
9904	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9905	{
9906	PVM pVM = pVCpu->CTX_SUFF(pVM);
9907	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9908
9909	Assert(PDMHasApic(pVM));
9910	Assert(u64MsrApicBase);
9911
9912	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9913	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9914
9915	/* Unalias any existing mapping. */
9916	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9917	AssertRCReturn(rc, rc);
9918
9919	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9920	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9921	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9922	AssertRCReturn(rc, rc);
9923
9924	/* Update the per-VCPU cache of the APIC base MSR. */
9925	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9926	return VINF_SUCCESS;
9927	}
9928
9929
9930	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9931	/**
9932	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9933	* nested-guest using hardware-assisted VMX.
9934	*
9935	* @param pVCpu The cross context virtual CPU structure.
9936	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9937	* @param pVmcsInfoGst The guest VMCS info. object.
9938	*/
9939	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9940	{
9941	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9942	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9943	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9944	Assert(pu64MsrBitmapNstGst);
9945	Assert(pu64MsrBitmapGst);
9946	Assert(pu64MsrBitmap);
9947
9948	/*
9949	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9950	* MSR that is intercepted by the guest is also intercepted while executing the
9951	* nested-guest using hardware-assisted VMX.
9952	*/
9953	uint32_t const cbFrag = sizeof(uint64_t);
9954	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9955	for (uint32_t i = 0; i <= cFrags; i++)
9956	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9957	}
9958
9959
9960	/**
9961	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9962	* hardware-assisted VMX execution of the nested-guest.
9963	*
9964	* For a guest, we don't modify these controls once we set up the VMCS and hence
9965	* this function is never called.
9966	*
9967	* For nested-guests since the guest hypervisor provides these controls on every
9968	* nested-guest VM-entry and could potentially change them everytime we need to
9969	* merge them before every nested-guest VM-entry.
9970	*
9971	* @returns VBox status code.
9972	* @param pVCpu The cross context virtual CPU structure.
9973	*/
9974	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9975	{
9976	PVM pVM = pVCpu->CTX_SUFF(pVM);
9977	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9978	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9979	Assert(pVmcsNstGst);
9980
9981	/*
9982	* Merge the controls with the requirements of the guest VMCS.
9983	*
9984	* We do not need to validate the nested-guest VMX features specified in the
9985	* nested-guest VMCS with the features supported by the physical CPU as it's
9986	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9987	*
9988	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9989	* guest are derived from the VMX features supported by the physical CPU.
9990	*/
9991
9992	/* Pin-based VM-execution controls. */
9993	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9994
9995	/* Processor-based VM-execution controls. */
9996	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9997	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9998	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9999	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10000	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10001
10002	/* Secondary processor-based VM-execution controls. */
10003	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10004	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10005	\| VMX_PROC_CTLS2_INVPCID
10006	\| VMX_PROC_CTLS2_RDTSCP
10007	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10008	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10009	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10010	\| VMX_PROC_CTLS2_VMFUNC));
10011
10012	/*
10013	* VM-entry controls:
10014	* These controls contains state that depends on the nested-guest state (primarily
10015	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10016	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10017	* properly continue executing the nested-guest if the EFER MSR changes but does not
10018	* cause a nested-guest VM-exits.
10019	*
10020	* VM-exit controls:
10021	* These controls specify the host state on return. We cannot use the controls from
10022	* the nested-hypervisor state as is as it would contain the guest state rather than
10023	* the host state. Since the host state is subject to change (e.g. preemption, trips
10024	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10025	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10026	*
10027	* VM-entry MSR-load:
10028	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10029	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10030	*
10031	* VM-exit MSR-store:
10032	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10033	* context back into the VM-exit MSR-store area.
10034	*
10035	* VM-exit MSR-load areas:
10036	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10037	* we can entirely ignore what the nested-hypervisor wants to load here.
10038	*/
10039
10040	/*
10041	* Exception bitmap.
10042	*
10043	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10044	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10045	* keep the code more flexible if intercepting exceptions become more dynamic in
10046	* the future we do it as part of exporting the nested-guest state.
10047	*/
10048	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10049
10050	/*
10051	* CR0/CR4 guest/host mask.
10052	*
10053	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10054	* must cause VM-exits, so we need to merge them here.
10055	*/
10056	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10057	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10058
10059	/*
10060	* Page-fault error-code mask and match.
10061	*
10062	* Although we require unrestricted guest execution (and thereby nested-paging) for
10063	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10064	* normally intercept #PFs, it might intercept them for debugging purposes.
10065	*
10066	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10067	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10068	*/
10069	uint32_t u32XcptPFMask;
10070	uint32_t u32XcptPFMatch;
10071	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10072	{
10073	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10074	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10075	}
10076	else
10077	{
10078	u32XcptPFMask = 0;
10079	u32XcptPFMatch = 0;
10080	}
10081
10082	/*
10083	* Pause-Loop exiting.
10084	*/
10085	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10086	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10087
10088	/*
10089	* I/O Bitmap.
10090	*
10091	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10092	* always intercept all I/O port accesses.
10093	*/
10094	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10095
10096	/*
10097	* APIC-access page.
10098	*
10099	* The APIC-access page address has already been initialized while setting up the
10100	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10101	* should not be on any consequence to the host or to the guest for that matter, but
10102	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10103	* emulation to keep it simple.
10104	*/
10105
10106	/*
10107	* Virtual-APIC page and TPR threshold.
10108	*
10109	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10110	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10111	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10112	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10113	*/
10114	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10115	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10116	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10117	{
10118	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10119
10120	/*
10121	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10122	* we would fail to obtain a valid host-physical address for its guest-physical
10123	* address.
10124	*
10125	* We currently do not support this scenario. Maybe in the future if there is a
10126	* pressing need we can explore making this particular set of conditions work.
10127	* Right now we just cause a VM-entry failure.
10128	*
10129	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10130	* so should not really failure at the moment.
10131	*/
10132	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10133	}
10134	else
10135	{
10136	/*
10137	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10138	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10139	* be taken care of by EPT/shadow paging.
10140	*/
10141	if (pVM->hm.s.fAllow64BitGuests)
10142	{
10143	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10144	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10145	}
10146	}
10147
10148	/*
10149	* Validate basic assumptions.
10150	*/
10151	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10152	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10153	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10154	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10155
10156	/*
10157	* Commit it to the nested-guest VMCS.
10158	*/
10159	int rc = VINF_SUCCESS;
10160	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10161	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10162	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10163	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10164	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10165	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10166	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10167	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10168	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10169	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10170	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10171	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10172	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10173	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10174	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10175	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10176	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10177	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10178	{
10179	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10180	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10181	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10182	}
10183	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10184	{
10185	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10186	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10187	}
10188	AssertRCReturn(rc, rc);
10189
10190	/*
10191	* Update the nested-guest VMCS cache.
10192	*/
10193	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10194	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10195	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10196	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10197	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10198	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10199	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10200	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10201	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10202
10203	/*
10204	* MSR bitmap.
10205	*
10206	* The MSR bitmap address has already been initialized while setting up the
10207	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10208	*/
10209	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10210	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10211
10212	return VINF_SUCCESS;
10213	}
10214	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10215
10216
10217	/**
10218	* Does the preparations before executing guest code in VT-x.
10219	*
10220	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10221	* recompiler/IEM. We must be cautious what we do here regarding committing
10222	* guest-state information into the VMCS assuming we assuredly execute the
10223	* guest in VT-x mode.
10224	*
10225	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10226	* the common-state (TRPM/forceflags), we must undo those changes so that the
10227	* recompiler/IEM can (and should) use them when it resumes guest execution.
10228	* Otherwise such operations must be done when we can no longer exit to ring-3.
10229	*
10230	* @returns Strict VBox status code (i.e. informational status codes too).
10231	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10232	* have been disabled.
10233	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10234	* double-fault into the guest.
10235	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10236	* dispatched directly.
10237	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10238	*
10239	* @param pVCpu The cross context virtual CPU structure.
10240	* @param pVmxTransient The VMX-transient structure.
10241	* @param fStepping Whether we are single-stepping the guest in the
10242	* hypervisor debugger. Makes us ignore some of the reasons
10243	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10244	* if event dispatching took place.
10245	*/
10246	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10247	{
10248	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10249
10250	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10251	if (pVmxTransient->fIsNestedGuest)
10252	{
10253	RT_NOREF2(pVCpu, fStepping);
10254	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10255	return VINF_EM_RESCHEDULE_REM;
10256	}
10257	#endif
10258
10259	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10260	PGMRZDynMapFlushAutoSet(pVCpu);
10261	#endif
10262
10263	/*
10264	* Check and process force flag actions, some of which might require us to go back to ring-3.
10265	*/
10266	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10267	if (rcStrict == VINF_SUCCESS)
10268	{ /* FFs don't get set all the time. */ }
10269	else
10270	return rcStrict;
10271
10272	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10273	/*
10274	* Switch to the nested-guest VMCS as we may have transitioned into executing
10275	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10276	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10277	*
10278	* We do this as late as possible to minimize (though not completely remove)
10279	* clearing/loading VMCS again due to premature trips to ring-3 above.
10280	*/
10281	if (pVmxTransient->fIsNestedGuest)
10282	{
10283	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10284	{
10285	/*
10286	* Ensure we have synced everything from the guest VMCS and also flag that
10287	* that we need to export the full (nested) guest-CPU context to the
10288	* nested-guest VMCS.
10289	*/
10290	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10291	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10292
10293	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10294	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10295	if (RT_LIKELY(rc == VINF_SUCCESS))
10296	{
10297	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10298	ASMSetFlags(fEFlags);
10299	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10300
10301	/*
10302	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10303	* flag that we need to update the host MSR values there. Even if we decide
10304	* in the future to share the VM-exit MSR-store area page with the guest,
10305	* if its content differs, we would have to update the host MSRs anyway.
10306	*/
10307	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10308	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10309	}
10310	else
10311	{
10312	ASMSetFlags(fEFlags);
10313	return rc;
10314	}
10315	}
10316
10317	/*
10318	* Merge guest VMCS controls with the nested-guest VMCS controls.
10319	*
10320	* Even if we have not executed the guest prior to this (e.g. when resuming
10321	* from a saved state), we should be okay with merging controls as we
10322	* initialize the guest VMCS controls as part of VM setup phase.
10323	*/
10324	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10325	{
10326	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10327	AssertRCReturn(rc, rc);
10328	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10329	}
10330	}
10331	#endif
10332
10333	/*
10334	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10335	* We look at the guest VMCS control here as we always set it when supported by
10336	* the physical CPU. Looking at the nested-guest control here would not be
10337	* possible because they are not merged yet.
10338	*/
10339	PVM pVM = pVCpu->CTX_SUFF(pVM);
10340	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10341	Assert(pVmcsInfo);
10342	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10343	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10344	&& PDMHasApic(pVM))
10345	{
10346	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10347	AssertRCReturn(rc, rc);
10348	}
10349
10350	/*
10351	* Evaluate events to be injected into the guest.
10352	*
10353	* Events in TRPM can be injected without inspecting the guest state.
10354	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10355	* guest to cause a VM-exit the next time they are ready to receive the event.
10356	*/
10357	if (TRPMHasTrap(pVCpu))
10358	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10359
10360	uint32_t fIntrState;
10361	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10362
10363	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10364	/*
10365	* While evaluating pending events if something failed (unlikely) or if we were
10366	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10367	*/
10368	if ( rcStrict != VINF_SUCCESS
10369	\|\| ( pVmxTransient->fIsNestedGuest
10370	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10371	return rcStrict;
10372	#endif
10373
10374	/*
10375	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10376	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10377	* also result in triple-faulting the VM.
10378	*
10379	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10380	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10381	*/
10382	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10383	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10384	{ /* likely */ }
10385	else
10386	{
10387	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10388	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10389	return rcStrict;
10390	}
10391
10392	/*
10393	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10394	* import CR3 themselves. We will need to update them here, as even as late as the above
10395	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10396	* the below force flags to be set.
10397	*/
10398	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10399	{
10400	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10401	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10402	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10403	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10404	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10405	}
10406	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10407	{
10408	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10409	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10410	}
10411
10412	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10413	/* Paranoia. */
10414	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10415	#endif
10416
10417	/*
10418	* No longjmps to ring-3 from this point on!!!
10419	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10420	* This also disables flushing of the R0-logger instance (if any).
10421	*/
10422	VMMRZCallRing3Disable(pVCpu);
10423
10424	/*
10425	* Export the guest state bits.
10426	*
10427	* We cannot perform longjmps while loading the guest state because we do not preserve the
10428	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10429	* CPU migration.
10430	*
10431	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10432	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10433	*/
10434	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10435	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10436	{ /* likely */ }
10437	else
10438	{
10439	VMMRZCallRing3Enable(pVCpu);
10440	return rcStrict;
10441	}
10442
10443	/*
10444	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10445	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10446	* preemption disabled for a while. Since this is purely to aid the
10447	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10448	* disable interrupt on NT.
10449	*
10450	* We need to check for force-flags that could've possible been altered since we last
10451	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10452	* see @bugref{6398}).
10453	*
10454	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10455	* to ring-3 before executing guest code.
10456	*/
10457	pVmxTransient->fEFlags = ASMIntDisableFlags();
10458
10459	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10460	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10461	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10462	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10463	{
10464	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10465	{
10466	pVCpu->hm.s.Event.fPending = false;
10467
10468	/*
10469	* We've injected any pending events. This is really the point of no return (to ring-3).
10470	*
10471	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10472	* returns from this function, so don't enable them here.
10473	*/
10474	return VINF_SUCCESS;
10475	}
10476
10477	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10478	rcStrict = VINF_EM_RAW_INTERRUPT;
10479	}
10480	else
10481	{
10482	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10483	rcStrict = VINF_EM_RAW_TO_R3;
10484	}
10485
10486	ASMSetFlags(pVmxTransient->fEFlags);
10487	VMMRZCallRing3Enable(pVCpu);
10488
10489	return rcStrict;
10490	}
10491
10492
10493	/**
10494	* Final preparations before executing guest code using hardware-assisted VMX.
10495	*
10496	* We can no longer get preempted to a different host CPU and there are no returns
10497	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10498	* failures), this function is not intended to fail sans unrecoverable hardware
10499	* errors.
10500	*
10501	* @param pVCpu The cross context virtual CPU structure.
10502	* @param pVmxTransient The VMX-transient structure.
10503	*
10504	* @remarks Called with preemption disabled.
10505	* @remarks No-long-jump zone!!!
10506	*/
10507	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10508	{
10509	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10510	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10511	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10512	Assert(!pVCpu->hm.s.Event.fPending);
10513
10514	/*
10515	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10516	*/
10517	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10518	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10519
10520	PVM pVM = pVCpu->CTX_SUFF(pVM);
10521	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10522
10523	if (!CPUMIsGuestFPUStateActive(pVCpu))
10524	{
10525	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10526	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10527	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10528	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10529	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10530	}
10531
10532	/*
10533	* Re-save the host state bits as we may've been preempted (only happens when
10534	* thread-context hooks are used or when the VM start function changes).
10535	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10536	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10537	* see @bugref{8432}.
10538	*
10539	* This may also happen when switching to/from a nested-guest VMCS without leaving
10540	* ring-0.
10541	*/
10542	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10543	{
10544	int rc = hmR0VmxExportHostState(pVCpu);
10545	AssertRC(rc);
10546	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10547	}
10548	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10549
10550	/*
10551	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10552	*/
10553	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10554	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10555	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10556
10557	/*
10558	* Store status of the shared guest/host debug state at the time of VM-entry.
10559	*/
10560	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10561	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10562	{
10563	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10564	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10565	}
10566	else
10567	#endif
10568	{
10569	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10570	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10571	}
10572
10573	/*
10574	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10575	* more than one conditional check. The post-run side of our code shall determine
10576	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10577	*/
10578	if (pVmcsInfo->pbVirtApic)
10579	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10580
10581	/*
10582	* Update the host MSRs values in the VM-exit MSR-load area.
10583	*/
10584	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10585	{
10586	if (pVmcsInfo->cExitMsrLoad > 0)
10587	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10588	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10589	}
10590
10591	/*
10592	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10593	* VMX-preemption timer based on the next virtual sync clock deadline.
10594	*/
10595	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10596	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10597	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10598	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10599	{
10600	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10601	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10602	}
10603
10604	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10605	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10606	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10607	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10608
10609	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10610
10611	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10612	as we're about to start executing the guest . */
10613
10614	/*
10615	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10616	*
10617	* This is done this late as updating the TSC offsetting/preemption timer above
10618	* figures out if we can skip intercepting RDTSCP by calculating the number of
10619	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10620	*/
10621	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10622	{
10623	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10624	{
10625	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10626	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10627	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10628	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10629	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10630	AssertRC(rc);
10631	}
10632	else
10633	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10634	}
10635
10636	#ifdef VBOX_STRICT
10637	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10638	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10639	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10640	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10641	#endif
10642
10643	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10644	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10645	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10646	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10647	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10648	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10649	#endif
10650	}
10651
10652
10653	/**
10654	* First C routine invoked after running guest code using hardware-assisted VMX.
10655	*
10656	* @param pVCpu The cross context virtual CPU structure.
10657	* @param pVmxTransient The VMX-transient structure.
10658	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10659	*
10660	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10661	*
10662	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10663	* unconditionally when it is safe to do so.
10664	*/
10665	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10666	{
10667	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10668
10669	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10670	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10671	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10672	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10673	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10674	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10675
10676	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10677	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10678	{
10679	uint64_t uGstTsc;
10680	if (!pVmxTransient->fIsNestedGuest)
10681	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10682	else
10683	{
10684	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10685	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10686	}
10687	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10688	}
10689
10690	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10691	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10692	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10693
10694	#if HC_ARCH_BITS == 64
10695	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10696	#endif
10697	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10698	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10699	and we need to leave it alone here. */
10700	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10701	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10702	#else
10703	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10704	#endif
10705	#ifdef VBOX_STRICT
10706	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10707	#endif
10708	Assert(!ASMIntAreEnabled());
10709	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10710	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10711
10712	/*
10713	* Save the basic VM-exit reason and check if the VM-entry failed.
10714	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10715	*/
10716	uint32_t uExitReason;
10717	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10718	AssertRC(rc);
10719	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10720	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10721
10722	/*
10723	* Check if VMLAUNCH/VMRESUME succeeded.
10724	* If this failed, we cause a guru meditation and cease further execution.
10725	*
10726	* However, if we are executing a nested-guest we might fail if we use the
10727	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10728	*/
10729	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10730	{
10731	/*
10732	* Update the VM-exit history array here even if the VM-entry failed due to:
10733	* - Invalid guest state.
10734	* - MSR loading.
10735	* - Machine-check event.
10736	*
10737	* In any of the above cases we will still have a "valid" VM-exit reason
10738	* despite @a fVMEntryFailed being false.
10739	*
10740	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10741	*
10742	* Note! We don't have CS or RIP at this point. Will probably address that later
10743	* by amending the history entry added here.
10744	*/
10745	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10746	UINT64_MAX, uHostTsc);
10747
10748	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10749	{
10750	VMMRZCallRing3Enable(pVCpu);
10751
10752	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10753	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10754
10755	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10756	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10757	AssertRC(rc);
10758	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10759	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10760	AssertRC(rc);
10761	#else
10762	/*
10763	* Import the guest-interruptibility state always as we need it while evaluating
10764	* injecting events on re-entry.
10765	*
10766	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10767	* checking for real-mode while exporting the state because all bits that cause
10768	* mode changes wrt CR0 are intercepted.
10769	*/
10770	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10771	AssertRC(rc);
10772	#endif
10773
10774	/*
10775	* Sync the TPR shadow with our APIC state.
10776	*/
10777	if ( !pVmxTransient->fIsNestedGuest
10778	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10779	{
10780	Assert(pVmcsInfo->pbVirtApic);
10781	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10782	{
10783	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10784	AssertRC(rc);
10785	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10786	}
10787	}
10788
10789	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10790	return;
10791	}
10792	}
10793	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10794	else if (pVmxTransient->fIsNestedGuest)
10795	{
10796	# if 0
10797	/*
10798	* Copy the VM-instruction error field to the guest VMCS.
10799	*/
10800	/** @todo NSTVMX: Verify we're using the fast path. */
10801	uint32_t u32RoVmInstrError;
10802	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10803	AssertRCReturn(rc, rc);
10804	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10805	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10806	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10807	# else
10808	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10809	# endif
10810	}
10811	#endif
10812	else
10813	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10814
10815	VMMRZCallRing3Enable(pVCpu);
10816	}
10817
10818
10819	/**
10820	* Runs the guest code using hardware-assisted VMX the normal way.
10821	*
10822	* @returns VBox status code.
10823	* @param pVCpu The cross context virtual CPU structure.
10824	* @param pcLoops Pointer to the number of executed loops.
10825	*/
10826	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10827	{
10828	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10829	Assert(pcLoops);
10830	Assert(*pcLoops <= cMaxResumeLoops);
10831
10832	VMXTRANSIENT VmxTransient;
10833	RT_ZERO(VmxTransient);
10834	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10835
10836	/* Paranoia. */
10837	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10838	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10839
10840	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10841	for (;;)
10842	{
10843	Assert(!HMR0SuspendPending());
10844	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10845	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10846
10847	/*
10848	* Preparatory work for running nested-guest code, this may force us to
10849	* return to ring-3.
10850	*
10851	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10852	*/
10853	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10854	if (rcStrict != VINF_SUCCESS)
10855	break;
10856
10857	/* Interrupts are disabled at this point! */
10858	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10859	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10860	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10861	/* Interrupts are re-enabled at this point! */
10862
10863	/*
10864	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10865	*/
10866	if (RT_SUCCESS(rcRun))
10867	{ /* very likely */ }
10868	else
10869	{
10870	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10871	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10872	return rcRun;
10873	}
10874
10875	/*
10876	* Profile the VM-exit.
10877	*/
10878	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10879	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10880	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10881	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10882	HMVMX_START_EXIT_DISPATCH_PROF();
10883
10884	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10885
10886	/*
10887	* Handle the VM-exit.
10888	*/
10889	#ifdef HMVMX_USE_FUNCTION_TABLE
10890	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10891	#else
10892	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10893	#endif
10894	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10895	if (rcStrict == VINF_SUCCESS)
10896	{
10897	if (++(*pcLoops) <= cMaxResumeLoops)
10898	continue;
10899	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10900	rcStrict = VINF_EM_RAW_INTERRUPT;
10901	}
10902	break;
10903	}
10904
10905	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10906	return rcStrict;
10907	}
10908
10909	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10910	/**
10911	* Runs the nested-guest code using hardware-assisted VMX.
10912	*
10913	* @returns VBox status code.
10914	* @param pVCpu The cross context virtual CPU structure.
10915	* @param pcLoops Pointer to the number of executed loops.
10916	*
10917	* @sa hmR0VmxRunGuestCodeNormal.
10918	*/
10919	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10920	{
10921	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10922	Assert(pcLoops);
10923	Assert(*pcLoops <= cMaxResumeLoops);
10924
10925	VMXTRANSIENT VmxTransient;
10926	RT_ZERO(VmxTransient);
10927	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10928	VmxTransient.fIsNestedGuest = true;
10929
10930	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10931	for (;;)
10932	{
10933	Assert(!HMR0SuspendPending());
10934	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10935	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10936
10937	/*
10938	* Preparatory work for running guest code, this may force us to
10939	* return to ring-3.
10940	*
10941	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10942	*/
10943	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10944	if (rcStrict != VINF_SUCCESS)
10945	break;
10946
10947	/* Interrupts are disabled at this point! */
10948	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10949	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10950	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10951	/* Interrupts are re-enabled at this point! */
10952
10953	/*
10954	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10955	*/
10956	if (RT_SUCCESS(rcRun))
10957	{ /* very likely */ }
10958	else
10959	{
10960	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10961	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10962	return rcRun;
10963	}
10964
10965	/*
10966	* Profile the VM-exit.
10967	*/
10968	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10969	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10970	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10971	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10972	HMVMX_START_EXIT_DISPATCH_PROF();
10973
10974	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10975
10976	/*
10977	* Handle the VM-exit.
10978	*/
10979	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10980	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10981	if ( rcStrict == VINF_SUCCESS
10982	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10983	{
10984	if (++(*pcLoops) <= cMaxResumeLoops)
10985	continue;
10986	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10987	rcStrict = VINF_EM_RAW_INTERRUPT;
10988	}
10989	break;
10990	}
10991
10992	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10993	return rcStrict;
10994	}
10995	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10996
10997
10998	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10999	* probes.
11000	*
11001	* The following few functions and associated structure contains the bloat
11002	* necessary for providing detailed debug events and dtrace probes as well as
11003	* reliable host side single stepping. This works on the principle of
11004	* "subclassing" the normal execution loop and workers. We replace the loop
11005	* method completely and override selected helpers to add necessary adjustments
11006	* to their core operation.
11007	*
11008	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11009	* any performance for debug and analysis features.
11010	*
11011	* @{
11012	*/
11013
11014	/**
11015	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11016	* the debug run loop.
11017	*/
11018	typedef struct VMXRUNDBGSTATE
11019	{
11020	/** The RIP we started executing at. This is for detecting that we stepped. */
11021	uint64_t uRipStart;
11022	/** The CS we started executing with. */
11023	uint16_t uCsStart;
11024
11025	/** Whether we've actually modified the 1st execution control field. */
11026	bool fModifiedProcCtls : 1;
11027	/** Whether we've actually modified the 2nd execution control field. */
11028	bool fModifiedProcCtls2 : 1;
11029	/** Whether we've actually modified the exception bitmap. */
11030	bool fModifiedXcptBitmap : 1;
11031
11032	/** We desire the modified the CR0 mask to be cleared. */
11033	bool fClearCr0Mask : 1;
11034	/** We desire the modified the CR4 mask to be cleared. */
11035	bool fClearCr4Mask : 1;
11036	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11037	uint32_t fCpe1Extra;
11038	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11039	uint32_t fCpe1Unwanted;
11040	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11041	uint32_t fCpe2Extra;
11042	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11043	uint32_t bmXcptExtra;
11044	/** The sequence number of the Dtrace provider settings the state was
11045	* configured against. */
11046	uint32_t uDtraceSettingsSeqNo;
11047	/** VM-exits to check (one bit per VM-exit). */
11048	uint32_t bmExitsToCheck[3];
11049
11050	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11051	uint32_t fProcCtlsInitial;
11052	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11053	uint32_t fProcCtls2Initial;
11054	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11055	uint32_t bmXcptInitial;
11056	} VMXRUNDBGSTATE;
11057	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11058	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11059
11060
11061	/**
11062	* Initializes the VMXRUNDBGSTATE structure.
11063	*
11064	* @param pVCpu The cross context virtual CPU structure of the
11065	* calling EMT.
11066	* @param pVmxTransient The VMX-transient structure.
11067	* @param pDbgState The debug state to initialize.
11068	*/
11069	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11070	{
11071	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11072	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11073
11074	pDbgState->fModifiedProcCtls = false;
11075	pDbgState->fModifiedProcCtls2 = false;
11076	pDbgState->fModifiedXcptBitmap = false;
11077	pDbgState->fClearCr0Mask = false;
11078	pDbgState->fClearCr4Mask = false;
11079	pDbgState->fCpe1Extra = 0;
11080	pDbgState->fCpe1Unwanted = 0;
11081	pDbgState->fCpe2Extra = 0;
11082	pDbgState->bmXcptExtra = 0;
11083	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11084	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11085	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11086	}
11087
11088
11089	/**
11090	* Updates the VMSC fields with changes requested by @a pDbgState.
11091	*
11092	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11093	* immediately before executing guest code, i.e. when interrupts are disabled.
11094	* We don't check status codes here as we cannot easily assert or return in the
11095	* latter case.
11096	*
11097	* @param pVCpu The cross context virtual CPU structure.
11098	* @param pVmxTransient The VMX-transient structure.
11099	* @param pDbgState The debug state.
11100	*/
11101	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11102	{
11103	/*
11104	* Ensure desired flags in VMCS control fields are set.
11105	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11106	*
11107	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11108	* there should be no stale data in pCtx at this point.
11109	*/
11110	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11111	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11112	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11113	{
11114	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11115	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11116	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11117	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11118	pDbgState->fModifiedProcCtls = true;
11119	}
11120
11121	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11122	{
11123	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11124	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11125	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11126	pDbgState->fModifiedProcCtls2 = true;
11127	}
11128
11129	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11130	{
11131	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11132	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11133	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11134	pDbgState->fModifiedXcptBitmap = true;
11135	}
11136
11137	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11138	{
11139	pVmcsInfo->u64Cr0Mask = 0;
11140	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11141	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11142	}
11143
11144	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11145	{
11146	pVmcsInfo->u64Cr4Mask = 0;
11147	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11148	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11149	}
11150
11151	NOREF(pVCpu);
11152	}
11153
11154
11155	/**
11156	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11157	* re-entry next time around.
11158	*
11159	* @returns Strict VBox status code (i.e. informational status codes too).
11160	* @param pVCpu The cross context virtual CPU structure.
11161	* @param pVmxTransient The VMX-transient structure.
11162	* @param pDbgState The debug state.
11163	* @param rcStrict The return code from executing the guest using single
11164	* stepping.
11165	*/
11166	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11167	VBOXSTRICTRC rcStrict)
11168	{
11169	/*
11170	* Restore VM-exit control settings as we may not reenter this function the
11171	* next time around.
11172	*/
11173	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11174
11175	/* We reload the initial value, trigger what we can of recalculations the
11176	next time around. From the looks of things, that's all that's required atm. */
11177	if (pDbgState->fModifiedProcCtls)
11178	{
11179	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11180	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11181	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11182	AssertRCReturn(rc2, rc2);
11183	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11184	}
11185
11186	/* We're currently the only ones messing with this one, so just restore the
11187	cached value and reload the field. */
11188	if ( pDbgState->fModifiedProcCtls2
11189	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11190	{
11191	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11192	AssertRCReturn(rc2, rc2);
11193	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11194	}
11195
11196	/* If we've modified the exception bitmap, we restore it and trigger
11197	reloading and partial recalculation the next time around. */
11198	if (pDbgState->fModifiedXcptBitmap)
11199	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11200
11201	return rcStrict;
11202	}
11203
11204
11205	/**
11206	* Configures VM-exit controls for current DBGF and DTrace settings.
11207	*
11208	* This updates @a pDbgState and the VMCS execution control fields to reflect
11209	* the necessary VM-exits demanded by DBGF and DTrace.
11210	*
11211	* @param pVCpu The cross context virtual CPU structure.
11212	* @param pVmxTransient The VMX-transient structure. May update
11213	* fUpdatedTscOffsettingAndPreemptTimer.
11214	* @param pDbgState The debug state.
11215	*/
11216	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11217	{
11218	/*
11219	* Take down the dtrace serial number so we can spot changes.
11220	*/
11221	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11222	ASMCompilerBarrier();
11223
11224	/*
11225	* We'll rebuild most of the middle block of data members (holding the
11226	* current settings) as we go along here, so start by clearing it all.
11227	*/
11228	pDbgState->bmXcptExtra = 0;
11229	pDbgState->fCpe1Extra = 0;
11230	pDbgState->fCpe1Unwanted = 0;
11231	pDbgState->fCpe2Extra = 0;
11232	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11233	pDbgState->bmExitsToCheck[i] = 0;
11234
11235	/*
11236	* Software interrupts (INT XXh) - no idea how to trigger these...
11237	*/
11238	PVM pVM = pVCpu->CTX_SUFF(pVM);
11239	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11240	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11241	{
11242	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11243	}
11244
11245	/*
11246	* INT3 breakpoints - triggered by #BP exceptions.
11247	*/
11248	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11249	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11250
11251	/*
11252	* Exception bitmap and XCPT events+probes.
11253	*/
11254	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11255	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11256	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11257
11258	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11259	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11260	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11261	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11262	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11263	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11264	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11265	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11266	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11267	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11268	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11269	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11270	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11271	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11272	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11273	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11274	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11275	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11276
11277	if (pDbgState->bmXcptExtra)
11278	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11279
11280	/*
11281	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11282	*
11283	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11284	* So, when adding/changing/removing please don't forget to update it.
11285	*
11286	* Some of the macros are picking up local variables to save horizontal space,
11287	* (being able to see it in a table is the lesser evil here).
11288	*/
11289	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11290	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11291	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11292	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11293	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11294	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11295	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11296	} else do { } while (0)
11297	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11298	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11299	{ \
11300	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11301	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11302	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11303	} else do { } while (0)
11304	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11305	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11306	{ \
11307	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11308	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11309	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11310	} else do { } while (0)
11311	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11312	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11313	{ \
11314	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11315	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11316	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11317	} else do { } while (0)
11318
11319	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11320	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11321	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11322	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11323	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11324
11325	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11326	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11327	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11328	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11329	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11330	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11331	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11332	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11333	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11334	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11335	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11336	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11337	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11338	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11339	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11340	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11341	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11342	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11343	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11344	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11345	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11346	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11347	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11349	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11350	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11351	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11352	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11353	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11354	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11355	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11356	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11357	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11358	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11359	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11360	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11361
11362	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11363	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11364	{
11365	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11366	\| CPUMCTX_EXTRN_APIC_TPR);
11367	AssertRC(rc);
11368
11369	#if 0 /** @todo fix me */
11370	pDbgState->fClearCr0Mask = true;
11371	pDbgState->fClearCr4Mask = true;
11372	#endif
11373	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11374	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11375	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11376	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11377	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11378	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11379	require clearing here and in the loop if we start using it. */
11380	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11381	}
11382	else
11383	{
11384	if (pDbgState->fClearCr0Mask)
11385	{
11386	pDbgState->fClearCr0Mask = false;
11387	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11388	}
11389	if (pDbgState->fClearCr4Mask)
11390	{
11391	pDbgState->fClearCr4Mask = false;
11392	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11393	}
11394	}
11395	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11396	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11397
11398	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11399	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11400	{
11401	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11402	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11403	}
11404	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11405	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11406
11407	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11408	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11409	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11410	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11411	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11412	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11413	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11414	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11415	#if 0 /** @todo too slow, fix handler. */
11416	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11417	#endif
11418	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11419
11420	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11421	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11422	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11423	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11424	{
11425	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11426	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11427	}
11428	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11430	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11431	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11432
11433	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11434	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11435	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11436	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11437	{
11438	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11439	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11440	}
11441	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11442	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11443	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11444	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11445
11446	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11447	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11448	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11449	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11450	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11451	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11452	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11453	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11454	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11455	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11456	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11457	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11458	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11459	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11460	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11461	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11462	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11463	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11464	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11465	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11466	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11467	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11468
11469	#undef IS_EITHER_ENABLED
11470	#undef SET_ONLY_XBM_IF_EITHER_EN
11471	#undef SET_CPE1_XBM_IF_EITHER_EN
11472	#undef SET_CPEU_XBM_IF_EITHER_EN
11473	#undef SET_CPE2_XBM_IF_EITHER_EN
11474
11475	/*
11476	* Sanitize the control stuff.
11477	*/
11478	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11479	if (pDbgState->fCpe2Extra)
11480	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11481	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11482	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11483	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11484	{
11485	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11486	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11487	}
11488
11489	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11490	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11491	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11492	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11493	}
11494
11495
11496	/**
11497	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11498	* appropriate.
11499	*
11500	* The caller has checked the VM-exit against the
11501	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11502	* already, so we don't have to do that either.
11503	*
11504	* @returns Strict VBox status code (i.e. informational status codes too).
11505	* @param pVCpu The cross context virtual CPU structure.
11506	* @param pVmxTransient The VMX-transient structure.
11507	* @param uExitReason The VM-exit reason.
11508	*
11509	* @remarks The name of this function is displayed by dtrace, so keep it short
11510	* and to the point. No longer than 33 chars long, please.
11511	*/
11512	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11513	{
11514	/*
11515	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11516	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11517	*
11518	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11519	* does. Must add/change/remove both places. Same ordering, please.
11520	*
11521	* Added/removed events must also be reflected in the next section
11522	* where we dispatch dtrace events.
11523	*/
11524	bool fDtrace1 = false;
11525	bool fDtrace2 = false;
11526	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11527	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11528	uint32_t uEventArg = 0;
11529	#define SET_EXIT(a_EventSubName) \
11530	do { \
11531	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11532	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11533	} while (0)
11534	#define SET_BOTH(a_EventSubName) \
11535	do { \
11536	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11537	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11538	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11539	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11540	} while (0)
11541	switch (uExitReason)
11542	{
11543	case VMX_EXIT_MTF:
11544	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11545
11546	case VMX_EXIT_XCPT_OR_NMI:
11547	{
11548	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11549	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11550	{
11551	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11552	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11553	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11554	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11555	{
11556	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11557	{
11558	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11559	uEventArg = pVmxTransient->uExitIntErrorCode;
11560	}
11561	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11562	switch (enmEvent1)
11563	{
11564	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11565	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11566	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11567	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11568	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11569	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11570	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11571	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11572	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11573	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11574	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11575	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11576	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11577	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11578	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11579	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11580	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11581	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11582	default: break;
11583	}
11584	}
11585	else
11586	AssertFailed();
11587	break;
11588
11589	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11590	uEventArg = idxVector;
11591	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11592	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11593	break;
11594	}
11595	break;
11596	}
11597
11598	case VMX_EXIT_TRIPLE_FAULT:
11599	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11600	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11601	break;
11602	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11603	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11604	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11605	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11606	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11607
11608	/* Instruction specific VM-exits: */
11609	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11610	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11611	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11612	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11613	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11614	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11615	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11616	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11617	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11618	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11619	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11620	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11621	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11622	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11623	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11624	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11625	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11626	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11627	case VMX_EXIT_MOV_CRX:
11628	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11629	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11630	SET_BOTH(CRX_READ);
11631	else
11632	SET_BOTH(CRX_WRITE);
11633	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11634	break;
11635	case VMX_EXIT_MOV_DRX:
11636	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11637	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11638	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11639	SET_BOTH(DRX_READ);
11640	else
11641	SET_BOTH(DRX_WRITE);
11642	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11643	break;
11644	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11645	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11646	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11647	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11648	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11649	case VMX_EXIT_GDTR_IDTR_ACCESS:
11650	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11651	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11652	{
11653	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11654	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11655	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11656	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11657	}
11658	break;
11659
11660	case VMX_EXIT_LDTR_TR_ACCESS:
11661	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11662	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11663	{
11664	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11665	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11666	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11667	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11668	}
11669	break;
11670
11671	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11672	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11673	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11674	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11675	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11676	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11677	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11678	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11679	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11680	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11681	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11682
11683	/* Events that aren't relevant at this point. */
11684	case VMX_EXIT_EXT_INT:
11685	case VMX_EXIT_INT_WINDOW:
11686	case VMX_EXIT_NMI_WINDOW:
11687	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11688	case VMX_EXIT_PREEMPT_TIMER:
11689	case VMX_EXIT_IO_INSTR:
11690	break;
11691
11692	/* Errors and unexpected events. */
11693	case VMX_EXIT_INIT_SIGNAL:
11694	case VMX_EXIT_SIPI:
11695	case VMX_EXIT_IO_SMI:
11696	case VMX_EXIT_SMI:
11697	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11698	case VMX_EXIT_ERR_MSR_LOAD:
11699	case VMX_EXIT_ERR_MACHINE_CHECK:
11700	break;
11701
11702	default:
11703	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11704	break;
11705	}
11706	#undef SET_BOTH
11707	#undef SET_EXIT
11708
11709	/*
11710	* Dtrace tracepoints go first. We do them here at once so we don't
11711	* have to copy the guest state saving and stuff a few dozen times.
11712	* Down side is that we've got to repeat the switch, though this time
11713	* we use enmEvent since the probes are a subset of what DBGF does.
11714	*/
11715	if (fDtrace1 \|\| fDtrace2)
11716	{
11717	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11718	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11719	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11720	switch (enmEvent1)
11721	{
11722	/** @todo consider which extra parameters would be helpful for each probe. */
11723	case DBGFEVENT_END: break;
11724	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11725	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11726	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11727	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11728	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11729	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11730	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11731	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11732	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11733	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11734	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11735	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11736	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11737	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11738	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11739	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11740	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11741	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11742	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11743	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11744	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11745	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11746	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11747	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11748	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11749	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11750	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11751	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11752	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11753	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11754	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11755	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11756	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11757	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11758	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11759	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11760	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11761	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11762	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11763	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11764	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11765	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11766	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11767	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11768	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11769	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11770	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11771	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11772	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11773	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11774	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11775	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11776	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11777	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11778	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11779	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11780	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11781	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11782	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11783	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11784	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11785	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11786	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11787	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11788	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11789	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11790	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11791	}
11792	switch (enmEvent2)
11793	{
11794	/** @todo consider which extra parameters would be helpful for each probe. */
11795	case DBGFEVENT_END: break;
11796	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11797	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11798	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11799	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11800	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11801	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11802	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11803	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11804	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11805	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11806	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11807	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11808	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11809	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11810	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11811	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11812	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11813	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11814	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11815	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11816	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11817	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11818	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11819	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11820	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11821	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11822	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11823	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11824	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11825	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11826	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11827	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11828	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11829	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11830	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11831	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11832	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11833	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11834	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11835	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11836	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11837	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11838	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11839	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11840	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11841	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11842	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11843	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11844	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11845	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11846	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11847	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11848	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11849	}
11850	}
11851
11852	/*
11853	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11854	* the DBGF call will do a full check).
11855	*
11856	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11857	* Note! If we have to events, we prioritize the first, i.e. the instruction
11858	* one, in order to avoid event nesting.
11859	*/
11860	PVM pVM = pVCpu->CTX_SUFF(pVM);
11861	if ( enmEvent1 != DBGFEVENT_END
11862	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11863	{
11864	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11865	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11866	if (rcStrict != VINF_SUCCESS)
11867	return rcStrict;
11868	}
11869	else if ( enmEvent2 != DBGFEVENT_END
11870	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11871	{
11872	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11873	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11874	if (rcStrict != VINF_SUCCESS)
11875	return rcStrict;
11876	}
11877
11878	return VINF_SUCCESS;
11879	}
11880
11881
11882	/**
11883	* Single-stepping VM-exit filtering.
11884	*
11885	* This is preprocessing the VM-exits and deciding whether we've gotten far
11886	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11887	* handling is performed.
11888	*
11889	* @returns Strict VBox status code (i.e. informational status codes too).
11890	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11891	* @param pVmxTransient The VMX-transient structure.
11892	* @param pDbgState The debug state.
11893	*/
11894	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11895	{
11896	/*
11897	* Expensive (saves context) generic dtrace VM-exit probe.
11898	*/
11899	uint32_t const uExitReason = pVmxTransient->uExitReason;
11900	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11901	{ /* more likely */ }
11902	else
11903	{
11904	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11905	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11906	AssertRC(rc);
11907	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11908	}
11909
11910	/*
11911	* Check for host NMI, just to get that out of the way.
11912	*/
11913	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11914	{ /* normally likely */ }
11915	else
11916	{
11917	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11918	AssertRCReturn(rc2, rc2);
11919	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11920	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11921	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11922	}
11923
11924	/*
11925	* Check for single stepping event if we're stepping.
11926	*/
11927	if (pVCpu->hm.s.fSingleInstruction)
11928	{
11929	switch (uExitReason)
11930	{
11931	case VMX_EXIT_MTF:
11932	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11933
11934	/* Various events: */
11935	case VMX_EXIT_XCPT_OR_NMI:
11936	case VMX_EXIT_EXT_INT:
11937	case VMX_EXIT_TRIPLE_FAULT:
11938	case VMX_EXIT_INT_WINDOW:
11939	case VMX_EXIT_NMI_WINDOW:
11940	case VMX_EXIT_TASK_SWITCH:
11941	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11942	case VMX_EXIT_APIC_ACCESS:
11943	case VMX_EXIT_EPT_VIOLATION:
11944	case VMX_EXIT_EPT_MISCONFIG:
11945	case VMX_EXIT_PREEMPT_TIMER:
11946
11947	/* Instruction specific VM-exits: */
11948	case VMX_EXIT_CPUID:
11949	case VMX_EXIT_GETSEC:
11950	case VMX_EXIT_HLT:
11951	case VMX_EXIT_INVD:
11952	case VMX_EXIT_INVLPG:
11953	case VMX_EXIT_RDPMC:
11954	case VMX_EXIT_RDTSC:
11955	case VMX_EXIT_RSM:
11956	case VMX_EXIT_VMCALL:
11957	case VMX_EXIT_VMCLEAR:
11958	case VMX_EXIT_VMLAUNCH:
11959	case VMX_EXIT_VMPTRLD:
11960	case VMX_EXIT_VMPTRST:
11961	case VMX_EXIT_VMREAD:
11962	case VMX_EXIT_VMRESUME:
11963	case VMX_EXIT_VMWRITE:
11964	case VMX_EXIT_VMXOFF:
11965	case VMX_EXIT_VMXON:
11966	case VMX_EXIT_MOV_CRX:
11967	case VMX_EXIT_MOV_DRX:
11968	case VMX_EXIT_IO_INSTR:
11969	case VMX_EXIT_RDMSR:
11970	case VMX_EXIT_WRMSR:
11971	case VMX_EXIT_MWAIT:
11972	case VMX_EXIT_MONITOR:
11973	case VMX_EXIT_PAUSE:
11974	case VMX_EXIT_GDTR_IDTR_ACCESS:
11975	case VMX_EXIT_LDTR_TR_ACCESS:
11976	case VMX_EXIT_INVEPT:
11977	case VMX_EXIT_RDTSCP:
11978	case VMX_EXIT_INVVPID:
11979	case VMX_EXIT_WBINVD:
11980	case VMX_EXIT_XSETBV:
11981	case VMX_EXIT_RDRAND:
11982	case VMX_EXIT_INVPCID:
11983	case VMX_EXIT_VMFUNC:
11984	case VMX_EXIT_RDSEED:
11985	case VMX_EXIT_XSAVES:
11986	case VMX_EXIT_XRSTORS:
11987	{
11988	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11989	AssertRCReturn(rc, rc);
11990	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11991	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11992	return VINF_EM_DBG_STEPPED;
11993	break;
11994	}
11995
11996	/* Errors and unexpected events: */
11997	case VMX_EXIT_INIT_SIGNAL:
11998	case VMX_EXIT_SIPI:
11999	case VMX_EXIT_IO_SMI:
12000	case VMX_EXIT_SMI:
12001	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12002	case VMX_EXIT_ERR_MSR_LOAD:
12003	case VMX_EXIT_ERR_MACHINE_CHECK:
12004	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12005	break;
12006
12007	default:
12008	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12009	break;
12010	}
12011	}
12012
12013	/*
12014	* Check for debugger event breakpoints and dtrace probes.
12015	*/
12016	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12017	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12018	{
12019	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12020	if (rcStrict != VINF_SUCCESS)
12021	return rcStrict;
12022	}
12023
12024	/*
12025	* Normal processing.
12026	*/
12027	#ifdef HMVMX_USE_FUNCTION_TABLE
12028	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12029	#else
12030	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12031	#endif
12032	}
12033
12034
12035	/**
12036	* Single steps guest code using hardware-assisted VMX.
12037	*
12038	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12039	* but single-stepping through the hypervisor debugger.
12040	*
12041	* @returns Strict VBox status code (i.e. informational status codes too).
12042	* @param pVCpu The cross context virtual CPU structure.
12043	* @param pcLoops Pointer to the number of executed loops.
12044	*
12045	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12046	*/
12047	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12048	{
12049	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12050	Assert(pcLoops);
12051	Assert(*pcLoops <= cMaxResumeLoops);
12052
12053	VMXTRANSIENT VmxTransient;
12054	RT_ZERO(VmxTransient);
12055	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12056
12057	/* Set HMCPU indicators. */
12058	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12059	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12060	pVCpu->hm.s.fDebugWantRdTscExit = false;
12061	pVCpu->hm.s.fUsingDebugLoop = true;
12062
12063	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12064	VMXRUNDBGSTATE DbgState;
12065	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12066	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12067
12068	/*
12069	* The loop.
12070	*/
12071	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12072	for (;;)
12073	{
12074	Assert(!HMR0SuspendPending());
12075	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12076	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12077	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12078
12079	/* Set up VM-execution controls the next two can respond to. */
12080	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12081
12082	/*
12083	* Preparatory work for running guest code, this may force us to
12084	* return to ring-3.
12085	*
12086	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12087	*/
12088	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12089	if (rcStrict != VINF_SUCCESS)
12090	break;
12091
12092	/* Interrupts are disabled at this point! */
12093	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12094
12095	/* Override any obnoxious code in the above two calls. */
12096	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12097
12098	/*
12099	* Finally execute the guest.
12100	*/
12101	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12102
12103	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12104	/* Interrupts are re-enabled at this point! */
12105
12106	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12107	if (RT_SUCCESS(rcRun))
12108	{ /* very likely */ }
12109	else
12110	{
12111	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12112	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12113	return rcRun;
12114	}
12115
12116	/* Profile the VM-exit. */
12117	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12118	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12119	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12120	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12121	HMVMX_START_EXIT_DISPATCH_PROF();
12122
12123	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12124
12125	/*
12126	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12127	*/
12128	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12129	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12130	if (rcStrict != VINF_SUCCESS)
12131	break;
12132	if (++(*pcLoops) > cMaxResumeLoops)
12133	{
12134	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12135	rcStrict = VINF_EM_RAW_INTERRUPT;
12136	break;
12137	}
12138
12139	/*
12140	* Stepping: Did the RIP change, if so, consider it a single step.
12141	* Otherwise, make sure one of the TFs gets set.
12142	*/
12143	if (fStepping)
12144	{
12145	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12146	AssertRC(rc);
12147	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12148	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12149	{
12150	rcStrict = VINF_EM_DBG_STEPPED;
12151	break;
12152	}
12153	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12154	}
12155
12156	/*
12157	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12158	*/
12159	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12160	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12161	}
12162
12163	/*
12164	* Clear the X86_EFL_TF if necessary.
12165	*/
12166	if (pVCpu->hm.s.fClearTrapFlag)
12167	{
12168	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12169	AssertRC(rc);
12170	pVCpu->hm.s.fClearTrapFlag = false;
12171	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12172	}
12173	/** @todo there seems to be issues with the resume flag when the monitor trap
12174	* flag is pending without being used. Seen early in bios init when
12175	* accessing APIC page in protected mode. */
12176
12177	/*
12178	* Restore VM-exit control settings as we may not re-enter this function the
12179	* next time around.
12180	*/
12181	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12182
12183	/* Restore HMCPU indicators. */
12184	pVCpu->hm.s.fUsingDebugLoop = false;
12185	pVCpu->hm.s.fDebugWantRdTscExit = false;
12186	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12187
12188	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12189	return rcStrict;
12190	}
12191
12192
12193	/** @} */
12194
12195
12196	/**
12197	* Checks if any expensive dtrace probes are enabled and we should go to the
12198	* debug loop.
12199	*
12200	* @returns true if we should use debug loop, false if not.
12201	*/
12202	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12203	{
12204	/* It's probably faster to OR the raw 32-bit counter variables together.
12205	Since the variables are in an array and the probes are next to one
12206	another (more or less), we have good locality. So, better read
12207	eight-nine cache lines ever time and only have one conditional, than
12208	128+ conditionals, right? */
12209	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12210	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12211	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12212	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12213	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12214	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12215	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12216	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12217	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12218	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12219	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12220	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12221	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12222	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12223	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12224	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12225	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12226	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12227	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12228	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12229	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12230	) != 0
12231	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12232	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12233	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12234	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12235	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12236	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12237	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12238	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12239	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12240	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12241	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12242	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12243	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12244	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12245	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12246	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12247	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12248	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12249	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12250	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12251	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12252	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12253	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12254	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12255	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12256	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12257	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12258	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12259	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12260	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12261	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12262	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12263	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12264	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12265	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12266	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12267	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12268	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12269	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12270	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12271	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12272	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12273	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12274	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12275	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12276	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12277	) != 0
12278	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12279	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12280	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12281	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12282	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12283	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12284	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12285	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12286	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12287	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12288	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12289	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12290	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12291	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12292	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12293	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12294	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12295	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12296	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12297	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12298	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12299	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12300	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12301	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12302	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12303	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12304	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12305	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12306	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12307	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12308	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12309	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12310	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12311	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12312	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12313	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12314	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12315	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12316	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12317	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12318	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12319	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12320	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12321	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12322	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12323	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12324	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12325	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12326	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12327	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12328	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12329	) != 0;
12330	}
12331
12332
12333	/**
12334	* Runs the guest using hardware-assisted VMX.
12335	*
12336	* @returns Strict VBox status code (i.e. informational status codes too).
12337	* @param pVCpu The cross context virtual CPU structure.
12338	*/
12339	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12340	{
12341	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12342	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12343	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12344	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12345
12346	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12347
12348	VBOXSTRICTRC rcStrict;
12349	uint32_t cLoops = 0;
12350	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12351	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12352	#else
12353	bool const fInNestedGuestMode = false;
12354	#endif
12355	if (!fInNestedGuestMode)
12356	{
12357	if ( !pVCpu->hm.s.fUseDebugLoop
12358	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12359	&& !DBGFIsStepping(pVCpu)
12360	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12361	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12362	else
12363	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12364	}
12365	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12366	else
12367	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12368
12369	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12370	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12371	#endif
12372
12373	if (rcStrict == VERR_EM_INTERPRETER)
12374	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12375	else if (rcStrict == VINF_EM_RESET)
12376	rcStrict = VINF_EM_TRIPLE_FAULT;
12377
12378	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12379	if (RT_FAILURE(rc2))
12380	{
12381	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12382	rcStrict = rc2;
12383	}
12384	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12385	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12386	return rcStrict;
12387	}
12388
12389
12390	#ifndef HMVMX_USE_FUNCTION_TABLE
12391	/**
12392	* Handles a guest VM-exit from hardware-assisted VMX execution.
12393	*
12394	* @returns Strict VBox status code (i.e. informational status codes too).
12395	* @param pVCpu The cross context virtual CPU structure.
12396	* @param pVmxTransient The VMX-transient structure.
12397	*/
12398	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12399	{
12400	#ifdef DEBUG_ramshankar
12401	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12402	do { \
12403	if (a_fSave != 0) \
12404	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12405	VBOXSTRICTRC rcStrict = a_CallExpr; \
12406	if (a_fSave != 0) \
12407	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12408	return rcStrict; \
12409	} while (0)
12410	#else
12411	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12412	#endif
12413	uint32_t const rcReason = pVmxTransient->uExitReason;
12414	switch (rcReason)
12415	{
12416	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12417	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12418	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12419	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12420	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12421	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12422	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12423	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12424	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12425	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12426	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12427	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12428	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12429	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12430	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12431	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12432	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12433	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12434	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12435	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12436	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12437	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12438	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12439	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12440	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12441	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12442	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12443	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12444	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12445	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12446	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12447	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12448	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12449	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12450	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12451	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12452	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12453	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12454	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12455	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12456	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12457	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12458	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12459	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12460	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12461	#else
12462	case VMX_EXIT_VMCLEAR:
12463	case VMX_EXIT_VMLAUNCH:
12464	case VMX_EXIT_VMPTRLD:
12465	case VMX_EXIT_VMPTRST:
12466	case VMX_EXIT_VMREAD:
12467	case VMX_EXIT_VMRESUME:
12468	case VMX_EXIT_VMWRITE:
12469	case VMX_EXIT_VMXOFF:
12470	case VMX_EXIT_VMXON:
12471	case VMX_EXIT_INVVPID:
12472	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12473	#endif
12474
12475	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12476	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12477	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12478	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12479	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12480	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12481	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12482	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12483	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12484
12485	case VMX_EXIT_INVEPT:
12486	case VMX_EXIT_VMFUNC:
12487	case VMX_EXIT_XSAVES:
12488	case VMX_EXIT_XRSTORS:
12489	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12490
12491	case VMX_EXIT_ENCLS:
12492	case VMX_EXIT_RDSEED:
12493	case VMX_EXIT_PML_FULL:
12494	default:
12495	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12496	}
12497	#undef VMEXIT_CALL_RET
12498	}
12499	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12500
12501
12502	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12503	/**
12504	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12505	*
12506	* @returns Strict VBox status code (i.e. informational status codes too).
12507	* @param pVCpu The cross context virtual CPU structure.
12508	* @param pVmxTransient The VMX-transient structure.
12509	*/
12510	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12511	{
12512	VBOXSTRICTRC rcStrict;
12513	uint32_t const uExitReason = pVmxTransient->uExitReason;
12514	switch (uExitReason)
12515	{
12516	case VMX_EXIT_EPT_MISCONFIG:
12517	rcStrict = hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12518	break;
12519
12520	case VMX_EXIT_EPT_VIOLATION:
12521	rcStrict = hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12522	break;
12523
12524	case VMX_EXIT_IO_INSTR:
12525	{
12526	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12527	AssertRCReturn(rc, rc);
12528
12529	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12530	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12531	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12532
12533	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
12534	uint8_t const cbAccess = s_aIOSizes[uIOSize];
12535	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
12536	{
12537	/*
12538	* IN/OUT instruction:
12539	* - Provides VM-exit instruction length.
12540	*
12541	* INS/OUTS instruction:
12542	* - Provides VM-exit instruction length.
12543	* - Provides Guest-linear address.
12544	* - Optionally provides VM-exit instruction info (depends on CPU feature).
12545	*/
12546	PVM pVM = pVCpu->CTX_SUFF(pVM);
12547	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12548	AssertRCReturn(rc, rc);
12549
12550	/* Make sure we don't use stale VMX-transient info. */
12551	pVmxTransient->ExitInstrInfo.u = 0;
12552	pVmxTransient->uGuestLinearAddr = 0;
12553
12554	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
12555	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12556	if (fIOString)
12557	{
12558	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12559	if (fVmxInsOutsInfo)
12560	{
12561	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
12562	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12563	}
12564	}
12565	AssertRCReturn(rc, rc);
12566
12567	VMXVEXITINFO ExitInfo;
12568	RT_ZERO(ExitInfo);
12569	ExitInfo.uReason = uExitReason;
12570	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12571	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12572	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12573	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
12574	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12575	}
12576	else
12577	rcStrict = hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
12578	break;
12579	}
12580
12581	case VMX_EXIT_HLT:
12582	{
12583	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12584	AssertRCReturn(rc, rc);
12585	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
12586	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12587	else
12588	rcStrict = hmR0VmxExitHlt(pVCpu, pVmxTransient);
12589	break;
12590	}
12591
12592	case VMX_EXIT_RDTSC:
12593	{
12594	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12595	AssertRCReturn(rc, rc);
12596	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12597	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12598	else
12599	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12600	break;
12601	}
12602
12603	case VMX_EXIT_RDTSCP:
12604	{
12605	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12606	AssertRCReturn(rc, rc);
12607	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12608	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12609	else
12610	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12611	break;
12612	}
12613
12614	/*
12615	* Instructions that cause VM-exits unconditionally.
12616	* - Provides VM-exit instruction length ONLY.
12617	*/
12618	case VMX_EXIT_CPUID:
12619	case VMX_EXIT_VMCALL:
12620	case VMX_EXIT_GETSEC:
12621	case VMX_EXIT_INVD:
12622	case VMX_EXIT_XSETBV:
12623	case VMX_EXIT_VMLAUNCH:
12624	case VMX_EXIT_VMRESUME:
12625	case VMX_EXIT_VMXOFF:
12626	{
12627	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12628	AssertRCReturn(rc, rc);
12629	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12630	break;
12631	}
12632
12633	/*
12634	* Instructions that cause VM-exits unconditionally.
12635	* - Provides VM-exit instruction length.
12636	* - Provides VM-exit information.
12637	* - Optionally provides VM-exit qualification.
12638	*
12639	* Since VM-exit qualification is 0 for all VM-exits where it is not
12640	* applicable, reading and passing it to the guest should produce
12641	* defined behavior.
12642	*
12643	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12644	*/
12645	case VMX_EXIT_INVEPT:
12646	case VMX_EXIT_INVVPID:
12647	case VMX_EXIT_VMCLEAR:
12648	case VMX_EXIT_VMPTRLD:
12649	case VMX_EXIT_VMPTRST:
12650	case VMX_EXIT_VMXON:
12651	{
12652	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12653	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12654	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12655	AssertRCReturn(rc, rc);
12656
12657	VMXVEXITINFO ExitInfo;
12658	RT_ZERO(ExitInfo);
12659	ExitInfo.uReason = uExitReason;
12660	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12661	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12662	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12663	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12664	break;
12665	}
12666
12667	case VMX_EXIT_INVLPG:
12668	{
12669	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12670	{
12671	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12672	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12673	AssertRCReturn(rc, rc);
12674
12675	VMXVEXITINFO ExitInfo;
12676	RT_ZERO(ExitInfo);
12677	ExitInfo.uReason = uExitReason;
12678	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12679	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12680	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12681	}
12682	else
12683	rcStrict = hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
12684	break;
12685	}
12686
12687	case VMX_EXIT_INVPCID:
12688	{
12689	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12690	{
12691	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12692	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12693	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12694	AssertRCReturn(rc, rc);
12695
12696	VMXVEXITINFO ExitInfo;
12697	RT_ZERO(ExitInfo);
12698	ExitInfo.uReason = uExitReason;
12699	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12700	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12701	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12702	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12703	}
12704	else
12705	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12706	break;
12707	}
12708
12709	case VMX_EXIT_APIC_ACCESS:
12710	case VMX_EXIT_XCPT_OR_NMI:
12711	case VMX_EXIT_MOV_CRX:
12712	case VMX_EXIT_EXT_INT:
12713	case VMX_EXIT_INT_WINDOW:
12714	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12715	case VMX_EXIT_MWAIT:
12716	case VMX_EXIT_MONITOR:
12717	case VMX_EXIT_PREEMPT_TIMER:
12718
12719	case VMX_EXIT_RDMSR:
12720	{
12721	uint32_t fMsrpm;
12722	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12723	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12724	else
12725	fMsrpm = VMXMSRPM_EXIT_RD;
12726
12727	if (fMsrpm & VMXMSRPM_EXIT_RD)
12728	{
12729	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12730	AssertRCReturn(rc, rc);
12731	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12732	}
12733	else
12734	rcStrict = hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
12735	break;
12736	}
12737
12738	case VMX_EXIT_WRMSR:
12739	{
12740	uint32_t fMsrpm;
12741	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12742	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12743	else
12744	fMsrpm = VMXMSRPM_EXIT_WR;
12745
12746	if (fMsrpm & VMXMSRPM_EXIT_WR)
12747	{
12748	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12749	AssertRCReturn(rc, rc);
12750	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12751	}
12752	else
12753	rcStrict = hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
12754	break;
12755	}
12756
12757	case VMX_EXIT_TASK_SWITCH:
12758	{
12759	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12760	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12761	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12762	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12763	AssertRCReturn(rc, rc);
12764
12765	VMXVEXITINFO ExitInfo;
12766	RT_ZERO(ExitInfo);
12767	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12768	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12769
12770	VMXVEXITEVENTINFO ExitEventInfo;
12771	RT_ZERO(ExitInfo);
12772	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
12773	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
12774
12775	rcStrict = IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
12776	break;
12777	}
12778
12779	case VMX_EXIT_WBINVD:
12780	{
12781	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
12782	{
12783	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12784	AssertRCReturn(rc, rc);
12785	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12786	}
12787	else
12788	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12789	break;
12790	}
12791
12792	case VMX_EXIT_MOV_DRX:
12793	case VMX_EXIT_RSM:
12794	case VMX_EXIT_MTF:
12795	case VMX_EXIT_PAUSE:
12796	case VMX_EXIT_GDTR_IDTR_ACCESS:
12797	case VMX_EXIT_LDTR_TR_ACCESS:
12798	case VMX_EXIT_RDRAND:
12799	case VMX_EXIT_RDPMC:
12800	case VMX_EXIT_VMREAD:
12801	case VMX_EXIT_VMWRITE:
12802	case VMX_EXIT_TRIPLE_FAULT:
12803	case VMX_EXIT_NMI_WINDOW:
12804	case VMX_EXIT_INIT_SIGNAL:
12805	case VMX_EXIT_SIPI:
12806	case VMX_EXIT_IO_SMI:
12807	case VMX_EXIT_SMI:
12808	case VMX_EXIT_ERR_MSR_LOAD:
12809	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12810	case VMX_EXIT_ERR_MACHINE_CHECK:
12811
12812	case VMX_EXIT_VMFUNC:
12813	case VMX_EXIT_XSAVES:
12814	case VMX_EXIT_XRSTORS:
12815
12816	case VMX_EXIT_ENCLS:
12817	case VMX_EXIT_RDSEED:
12818	case VMX_EXIT_PML_FULL:
12819	default:
12820	rcStrict = hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12821	break;
12822	}
12823
12824	return rcStrict;
12825	}
12826	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12827
12828
12829	#ifdef VBOX_STRICT
12830	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12831	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12832	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12833
12834	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12835	do { \
12836	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12837	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12838	} while (0)
12839
12840	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12841	do { \
12842	AssertPtr((a_pVCpu)); \
12843	AssertPtr((a_pVmxTransient)); \
12844	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12845	Assert((a_pVmxTransient)->pVmcsInfo); \
12846	Assert(ASMIntAreEnabled()); \
12847	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12848	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12849	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12850	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12851	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12852	HMVMX_ASSERT_PREEMPT_CPUID(); \
12853	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12854	} while (0)
12855
12856	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12857	do { \
12858	Log4Func(("\n")); \
12859	} while (0)
12860	#else
12861	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12862	do { \
12863	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12864	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12865	} while (0)
12866	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12867	#endif
12868
12869
12870	/**
12871	* Advances the guest RIP by the specified number of bytes.
12872	*
12873	* @param pVCpu The cross context virtual CPU structure.
12874	* @param cbInstr Number of bytes to advance the RIP by.
12875	*
12876	* @remarks No-long-jump zone!!!
12877	*/
12878	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12879	{
12880	/* Advance the RIP. */
12881	pVCpu->cpum.GstCtx.rip += cbInstr;
12882	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12883
12884	/* Update interrupt inhibition. */
12885	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12886	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12887	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12888	}
12889
12890
12891	/**
12892	* Advances the guest RIP after reading it from the VMCS.
12893	*
12894	* @returns VBox status code, no informational status codes.
12895	* @param pVCpu The cross context virtual CPU structure.
12896	* @param pVmxTransient The VMX-transient structure.
12897	*
12898	* @remarks No-long-jump zone!!!
12899	*/
12900	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12901	{
12902	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12903	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12904	AssertRCReturn(rc, rc);
12905
12906	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12907	return VINF_SUCCESS;
12908	}
12909
12910
12911	/**
12912	* Handle a condition that occurred while delivering an event through the guest
12913	* IDT.
12914	*
12915	* @returns Strict VBox status code (i.e. informational status codes too).
12916	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12917	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12918	* to continue execution of the guest which will delivery the \#DF.
12919	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12920	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12921	*
12922	* @param pVCpu The cross context virtual CPU structure.
12923	* @param pVmxTransient The VMX-transient structure.
12924	*
12925	* @remarks No-long-jump zone!!!
12926	*/
12927	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12928	{
12929	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12930
12931	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12932	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12933	AssertRCReturn(rc2, rc2);
12934
12935	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12936	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12937	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12938	{
12939	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12940	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12941
12942	/*
12943	* If the event was a software interrupt (generated with INT n) or a software exception
12944	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12945	* can handle the VM-exit and continue guest execution which will re-execute the
12946	* instruction rather than re-injecting the exception, as that can cause premature
12947	* trips to ring-3 before injection and involve TRPM which currently has no way of
12948	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12949	* the problem).
12950	*/
12951	IEMXCPTRAISE enmRaise;
12952	IEMXCPTRAISEINFO fRaiseInfo;
12953	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12954	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12955	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12956	{
12957	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12958	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12959	}
12960	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12961	{
12962	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12963	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12964	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12965	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12966	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12967	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12968
12969	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12970
12971	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12972	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12973	{
12974	pVmxTransient->fVectoringPF = true;
12975	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12976	}
12977	}
12978	else
12979	{
12980	/*
12981	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12982	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12983	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12984	*/
12985	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12986	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12987	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12988	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12989	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12990	}
12991
12992	/*
12993	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12994	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12995	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12996	* subsequent VM-entry would fail.
12997	*
12998	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12999	*/
13000	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
13001	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13002	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13003	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13004	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13005	{
13006	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
13007	}
13008
13009	switch (enmRaise)
13010	{
13011	case IEMXCPTRAISE_CURRENT_XCPT:
13012	{
13013	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13014	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13015	Assert(rcStrict == VINF_SUCCESS);
13016	break;
13017	}
13018
13019	case IEMXCPTRAISE_PREV_EVENT:
13020	{
13021	uint32_t u32ErrCode;
13022	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13023	{
13024	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13025	AssertRCReturn(rc2, rc2);
13026	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13027	}
13028	else
13029	u32ErrCode = 0;
13030
13031	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13032	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13033	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13034	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13035
13036	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13037	pVCpu->hm.s.Event.u32ErrCode));
13038	Assert(rcStrict == VINF_SUCCESS);
13039	break;
13040	}
13041
13042	case IEMXCPTRAISE_REEXEC_INSTR:
13043	Assert(rcStrict == VINF_SUCCESS);
13044	break;
13045
13046	case IEMXCPTRAISE_DOUBLE_FAULT:
13047	{
13048	/*
13049	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13050	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13051	*/
13052	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13053	{
13054	pVmxTransient->fVectoringDoublePF = true;
13055	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13056	pVCpu->cpum.GstCtx.cr2));
13057	rcStrict = VINF_SUCCESS;
13058	}
13059	else
13060	{
13061	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13062	hmR0VmxSetPendingXcptDF(pVCpu);
13063	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13064	uIdtVector, uExitVector));
13065	rcStrict = VINF_HM_DOUBLE_FAULT;
13066	}
13067	break;
13068	}
13069
13070	case IEMXCPTRAISE_TRIPLE_FAULT:
13071	{
13072	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13073	rcStrict = VINF_EM_RESET;
13074	break;
13075	}
13076
13077	case IEMXCPTRAISE_CPU_HANG:
13078	{
13079	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13080	rcStrict = VERR_EM_GUEST_CPU_HANG;
13081	break;
13082	}
13083
13084	default:
13085	{
13086	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13087	rcStrict = VERR_VMX_IPE_2;
13088	break;
13089	}
13090	}
13091	}
13092	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13093	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13094	&& uExitVector != X86_XCPT_DF
13095	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13096	{
13097	/*
13098	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13099	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13100	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13101	*/
13102	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
13103	{
13104	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
13105	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
13106	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
13107	}
13108	}
13109
13110	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13111	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13112	return rcStrict;
13113	}
13114
13115
13116	/** @name VM-exit handlers.
13117	* @{
13118	*/
13119	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13120	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13121	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13122
13123	/**
13124	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13125	*/
13126	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13127	{
13128	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13130	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13131	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13132	return VINF_SUCCESS;
13133	return VINF_EM_RAW_INTERRUPT;
13134	}
13135
13136
13137	/**
13138	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13139	*/
13140	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13141	{
13142	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13143	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13144
13145	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13146	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13147	AssertRCReturn(rc, rc);
13148
13149	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13150	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13151	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13152	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13153
13154	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13155	{
13156	/*
13157	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13158	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13159	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13160	*
13161	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13162	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13163	*/
13164	VMXDispatchHostNmi();
13165	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
13166	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13167	return VINF_SUCCESS;
13168	}
13169
13170	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13171	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13172	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
13173	{ /* likely */ }
13174	else
13175	{
13176	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
13177	rcStrictRc1 = VINF_SUCCESS;
13178	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13179	return rcStrictRc1;
13180	}
13181
13182	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13183	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13184	switch (uIntType)
13185	{
13186	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13187	Assert(uVector == X86_XCPT_DB);
13188	RT_FALL_THRU();
13189	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13190	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13191	RT_FALL_THRU();
13192	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13193	{
13194	/*
13195	* If there's any exception caused as a result of event injection, the resulting
13196	* secondary/final execption will be pending, we shall continue guest execution
13197	* after injecting the event. The page-fault case is complicated and we manually
13198	* handle any currently pending event in hmR0VmxExitXcptPF.
13199	*/
13200	if (!pVCpu->hm.s.Event.fPending)
13201	{ /* likely */ }
13202	else if (uVector != X86_XCPT_PF)
13203	{
13204	rc = VINF_SUCCESS;
13205	break;
13206	}
13207
13208	switch (uVector)
13209	{
13210	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13211	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13212	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13213	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13214	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13215	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13216
13217	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13218	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13219	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13220	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13221	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13222	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13223	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13224	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13225	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13226	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13227	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13228	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13229	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13230	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13231	default:
13232	{
13233	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13234	if (pVmcsInfo->RealMode.fRealOnV86Active)
13235	{
13236	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13237	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13238	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13239
13240	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13241	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13242	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13243	AssertRCReturn(rc, rc);
13244	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13245	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13246	0 /* GCPtrFaultAddress */);
13247	}
13248	else
13249	{
13250	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13251	pVCpu->hm.s.u32HMError = uVector;
13252	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
13253	}
13254	break;
13255	}
13256	}
13257	break;
13258	}
13259
13260	default:
13261	{
13262	pVCpu->hm.s.u32HMError = uExitIntInfo;
13263	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13264	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13265	break;
13266	}
13267	}
13268	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13269	return rc;
13270	}
13271
13272
13273	/**
13274	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13275	*/
13276	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13277	{
13278	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13279
13280	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13281	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13282	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13283	AssertRCReturn(rc, rc);
13284
13285	/* Evaluate and deliver pending events and resume guest execution. */
13286	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13287	return VINF_SUCCESS;
13288	}
13289
13290
13291	/**
13292	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13293	*/
13294	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13295	{
13296	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13297
13298	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13299	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13300	{
13301	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13302	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13303	}
13304
13305	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13306
13307	/*
13308	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13309	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13310	*/
13311	uint32_t fIntrState;
13312	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13313	AssertRCReturn(rc, rc);
13314	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13315	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13316	{
13317	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13318	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13319
13320	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13321	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13322	AssertRCReturn(rc, rc);
13323	}
13324
13325	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13326	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13327	AssertRCReturn(rc, rc);
13328
13329	/* Evaluate and deliver pending events and resume guest execution. */
13330	return VINF_SUCCESS;
13331	}
13332
13333
13334	/**
13335	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13336	*/
13337	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13338	{
13339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13340	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13341	}
13342
13343
13344	/**
13345	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13346	*/
13347	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13348	{
13349	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13350	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13351	}
13352
13353
13354	/**
13355	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13356	*/
13357	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13358	{
13359	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13360
13361	/*
13362	* Get the state we need and update the exit history entry.
13363	*/
13364	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13365	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13366	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13367	AssertRCReturn(rc, rc);
13368
13369	VBOXSTRICTRC rcStrict;
13370	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13371	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13372	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13373	if (!pExitRec)
13374	{
13375	/*
13376	* Regular CPUID instruction execution.
13377	*/
13378	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13379	if (rcStrict == VINF_SUCCESS)
13380	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13381	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13382	{
13383	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13384	rcStrict = VINF_SUCCESS;
13385	}
13386	}
13387	else
13388	{
13389	/*
13390	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13391	*/
13392	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13393	AssertRCReturn(rc2, rc2);
13394
13395	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13396	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13397
13398	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13399	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13400
13401	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13402	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13403	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13404	}
13405	return rcStrict;
13406	}
13407
13408
13409	/**
13410	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13411	*/
13412	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13413	{
13414	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13415
13416	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13417	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13418	AssertRCReturn(rc, rc);
13419
13420	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13421	return VINF_EM_RAW_EMULATE_INSTR;
13422
13423	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13424	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13425	}
13426
13427
13428	/**
13429	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13430	*/
13431	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13432	{
13433	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13434
13435	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13436	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13437	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13438	AssertRCReturn(rc, rc);
13439
13440	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13441	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13442	{
13443	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13444	we must reset offsetting on VM-entry. See @bugref{6634}. */
13445	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13446	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13447	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13448	}
13449	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13450	{
13451	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13452	rcStrict = VINF_SUCCESS;
13453	}
13454	return rcStrict;
13455	}
13456
13457
13458	/**
13459	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13460	*/
13461	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13462	{
13463	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13464
13465	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13466	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13467	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13468	AssertRCReturn(rc, rc);
13469
13470	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13471	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13472	{
13473	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13474	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13475	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13476	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13477	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13478	}
13479	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13480	{
13481	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13482	rcStrict = VINF_SUCCESS;
13483	}
13484	return rcStrict;
13485	}
13486
13487
13488	/**
13489	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13490	*/
13491	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13492	{
13493	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13494
13495	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13496	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13497	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13498	AssertRCReturn(rc, rc);
13499
13500	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13501	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13502	if (RT_LIKELY(rc == VINF_SUCCESS))
13503	{
13504	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13505	Assert(pVmxTransient->cbInstr == 2);
13506	}
13507	else
13508	{
13509	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13510	rc = VERR_EM_INTERPRETER;
13511	}
13512	return rc;
13513	}
13514
13515
13516	/**
13517	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13518	*/
13519	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13520	{
13521	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13522
13523	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13524	if (EMAreHypercallInstructionsEnabled(pVCpu))
13525	{
13526	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13527	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13528	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13529	AssertRCReturn(rc, rc);
13530
13531	/* Perform the hypercall. */
13532	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13533	if (rcStrict == VINF_SUCCESS)
13534	{
13535	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13536	AssertRCReturn(rc, rc);
13537	}
13538	else
13539	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13540	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13541	\|\| RT_FAILURE(rcStrict));
13542
13543	/* If the hypercall changes anything other than guest's general-purpose registers,
13544	we would need to reload the guest changed bits here before VM-entry. */
13545	}
13546	else
13547	Log4Func(("Hypercalls not enabled\n"));
13548
13549	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13550	if (RT_FAILURE(rcStrict))
13551	{
13552	hmR0VmxSetPendingXcptUD(pVCpu);
13553	rcStrict = VINF_SUCCESS;
13554	}
13555
13556	return rcStrict;
13557	}
13558
13559
13560	/**
13561	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13562	*/
13563	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13564	{
13565	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13566	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13567
13568	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13569	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13570	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13571	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13572	AssertRCReturn(rc, rc);
13573
13574	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13575
13576	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13577	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13578	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13579	{
13580	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13581	rcStrict = VINF_SUCCESS;
13582	}
13583	else
13584	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13585	VBOXSTRICTRC_VAL(rcStrict)));
13586	return rcStrict;
13587	}
13588
13589
13590	/**
13591	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13592	*/
13593	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13594	{
13595	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13596
13597	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13598	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13599	AssertRCReturn(rc, rc);
13600
13601	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13602	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13603	if (RT_LIKELY(rc == VINF_SUCCESS))
13604	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13605	else
13606	{
13607	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13608	rc = VERR_EM_INTERPRETER;
13609	}
13610	return rc;
13611	}
13612
13613
13614	/**
13615	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13616	*/
13617	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13618	{
13619	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13620
13621	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13622	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13623	AssertRCReturn(rc, rc);
13624
13625	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13626	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13627	rc = VBOXSTRICTRC_VAL(rc2);
13628	if (RT_LIKELY( rc == VINF_SUCCESS
13629	\|\| rc == VINF_EM_HALT))
13630	{
13631	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13632	AssertRCReturn(rc3, rc3);
13633
13634	if ( rc == VINF_EM_HALT
13635	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13636	rc = VINF_SUCCESS;
13637	}
13638	else
13639	{
13640	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13641	rc = VERR_EM_INTERPRETER;
13642	}
13643	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13644	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13645	return rc;
13646	}
13647
13648
13649	/**
13650	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13651	*/
13652	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13653	{
13654	/*
13655	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13656	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13657	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13658	* VMX root operation. If we get here, something funny is going on.
13659	*
13660	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13661	*/
13662	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13663	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13664	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13665	}
13666
13667
13668	/**
13669	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13670	*/
13671	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13672	{
13673	/*
13674	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13675	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13676	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13677	* an SMI. If we get here, something funny is going on.
13678	*
13679	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13680	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13681	*/
13682	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13683	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13684	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13685	}
13686
13687
13688	/**
13689	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13690	*/
13691	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13692	{
13693	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13694	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13695	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13696	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13697	}
13698
13699
13700	/**
13701	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13702	*/
13703	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13704	{
13705	/*
13706	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13707	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13708	* See Intel spec. 25.3 "Other Causes of VM-exits".
13709	*/
13710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13711	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13712	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13713	}
13714
13715
13716	/**
13717	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13718	* VM-exit.
13719	*/
13720	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13721	{
13722	/*
13723	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13724	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13725	*
13726	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13727	* See Intel spec. "23.8 Restrictions on VMX operation".
13728	*/
13729	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13730	return VINF_SUCCESS;
13731	}
13732
13733
13734	/**
13735	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13736	* VM-exit.
13737	*/
13738	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13739	{
13740	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13741	return VINF_EM_RESET;
13742	}
13743
13744
13745	/**
13746	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13747	*/
13748	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13749	{
13750	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13751
13752	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13753	AssertRCReturn(rc, rc);
13754
13755	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13756	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13757	rc = VINF_SUCCESS;
13758	else
13759	rc = VINF_EM_HALT;
13760
13761	if (rc != VINF_SUCCESS)
13762	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13763	return rc;
13764	}
13765
13766
13767	/**
13768	* VM-exit handler for instructions that result in a \#UD exception delivered to
13769	* the guest.
13770	*/
13771	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13772	{
13773	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13774	hmR0VmxSetPendingXcptUD(pVCpu);
13775	return VINF_SUCCESS;
13776	}
13777
13778
13779	/**
13780	* VM-exit handler for expiry of the VMX-preemption timer.
13781	*/
13782	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13783	{
13784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13785
13786	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13787	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13788
13789	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13790	PVM pVM = pVCpu->CTX_SUFF(pVM);
13791	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13793	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13794	}
13795
13796
13797	/**
13798	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13799	*/
13800	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13801	{
13802	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13803
13804	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13805	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13806	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13807	AssertRCReturn(rc, rc);
13808
13809	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13810	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13811	: HM_CHANGED_RAISED_XCPT_MASK);
13812
13813	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13814	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13815
13816	return rcStrict;
13817	}
13818
13819
13820	/**
13821	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13822	*/
13823	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13824	{
13825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13826	/** @todo Use VM-exit instruction information. */
13827	return VERR_EM_INTERPRETER;
13828	}
13829
13830
13831	/**
13832	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13833	* Error VM-exit.
13834	*/
13835	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13836	{
13837	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13838	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13839	AssertRCReturn(rc, rc);
13840
13841	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13842	if (RT_FAILURE(rc))
13843	return rc;
13844
13845	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13846	NOREF(uInvalidReason);
13847
13848	#ifdef VBOX_STRICT
13849	uint32_t fIntrState;
13850	RTHCUINTREG uHCReg;
13851	uint64_t u64Val;
13852	uint32_t u32Val;
13853	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13854	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13855	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13856	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13857	AssertRCReturn(rc, rc);
13858
13859	Log4(("uInvalidReason %u\n", uInvalidReason));
13860	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13861	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13862	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13863	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13864
13865	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13866	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13867	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13868	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13869	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13870	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13871	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13872	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13873	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13874	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13875	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13876	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13877
13878	hmR0DumpRegs(pVCpu);
13879	#endif
13880
13881	return VERR_VMX_INVALID_GUEST_STATE;
13882	}
13883
13884
13885	/**
13886	* VM-exit handler for VM-entry failure due to an MSR-load
13887	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13888	*/
13889	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13890	{
13891	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13892	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13893	}
13894
13895
13896	/**
13897	* VM-exit handler for VM-entry failure due to a machine-check event
13898	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13899	*/
13900	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13901	{
13902	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13903	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13904	}
13905
13906
13907	/**
13908	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13909	* theory.
13910	*/
13911	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13912	{
13913	RT_NOREF2(pVCpu, pVmxTransient);
13914	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13915	return VERR_VMX_UNDEFINED_EXIT_CODE;
13916	}
13917
13918
13919	/**
13920	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13921	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13922	* Conditional VM-exit.
13923	*/
13924	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13925	{
13926	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13927
13928	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13930	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13931	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13932	return VERR_EM_INTERPRETER;
13933	AssertMsgFailed(("Unexpected XDTR access\n"));
13934	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13935	}
13936
13937
13938	/**
13939	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13940	*/
13941	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13942	{
13943	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13944
13945	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13946	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13947	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13948	return VERR_EM_INTERPRETER;
13949	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13950	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13951	}
13952
13953
13954	/**
13955	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13956	*/
13957	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13958	{
13959	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13960
13961	/** @todo Optimize this: We currently drag in in the whole MSR state
13962	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13963	* MSRs required. That would require changes to IEM and possibly CPUM too.
13964	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13965	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13966	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13967	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13968	switch (idMsr)
13969	{
13970	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13971	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13972	}
13973
13974	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13975	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13976	AssertRCReturn(rc, rc);
13977
13978	Log4Func(("ecx=%#RX32\n", idMsr));
13979
13980	#ifdef VBOX_STRICT
13981	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13982	{
13983	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13984	&& idMsr != MSR_K6_EFER)
13985	{
13986	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13987	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13988	}
13989	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13990	{
13991	Assert(pVmcsInfo->pvMsrBitmap);
13992	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13993	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13994	{
13995	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13996	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13997	}
13998	}
13999	}
14000	#endif
14001
14002	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14004	if (rcStrict == VINF_SUCCESS)
14005	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14006	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14007	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14008	{
14009	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14010	rcStrict = VINF_SUCCESS;
14011	}
14012	else
14013	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14014
14015	return rcStrict;
14016	}
14017
14018
14019	/**
14020	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14021	*/
14022	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14023	{
14024	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14025
14026	/** @todo Optimize this: We currently drag in in the whole MSR state
14027	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14028	* MSRs required. That would require changes to IEM and possibly CPUM too.
14029	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14030	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14031	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14032
14033	/*
14034	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14035	* Although we don't need to fetch the base as it will be overwritten shortly, while
14036	* loading guest-state we would also load the entire segment register including limit
14037	* and attributes and thus we need to load them here.
14038	*/
14039	switch (idMsr)
14040	{
14041	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14042	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14043	}
14044
14045	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14046	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14047	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14048	AssertRCReturn(rc, rc);
14049
14050	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14051
14052	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14053	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14054
14055	if (rcStrict == VINF_SUCCESS)
14056	{
14057	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14058
14059	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14060	if ( idMsr == MSR_IA32_APICBASE
14061	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14062	&& idMsr <= MSR_IA32_X2APIC_END))
14063	{
14064	/*
14065	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14066	* When full APIC register virtualization is implemented we'll have to make
14067	* sure APIC state is saved from the VMCS before IEM changes it.
14068	*/
14069	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14070	}
14071	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14072	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14073	else if (idMsr == MSR_K6_EFER)
14074	{
14075	/*
14076	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14077	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14078	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14079	*/
14080	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14081	}
14082
14083	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14084	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14085	{
14086	switch (idMsr)
14087	{
14088	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14089	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14090	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14091	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14092	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14093	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14094	default:
14095	{
14096	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14097	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14098	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14099	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14100	break;
14101	}
14102	}
14103	}
14104	#ifdef VBOX_STRICT
14105	else
14106	{
14107	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14108	switch (idMsr)
14109	{
14110	case MSR_IA32_SYSENTER_CS:
14111	case MSR_IA32_SYSENTER_EIP:
14112	case MSR_IA32_SYSENTER_ESP:
14113	case MSR_K8_FS_BASE:
14114	case MSR_K8_GS_BASE:
14115	{
14116	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14117	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14118	}
14119
14120	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14121	default:
14122	{
14123	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14124	{
14125	/* EFER MSR writes are always intercepted. */
14126	if (idMsr != MSR_K6_EFER)
14127	{
14128	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14129	idMsr));
14130	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14131	}
14132	}
14133
14134	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14135	{
14136	Assert(pVmcsInfo->pvMsrBitmap);
14137	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14138	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14139	{
14140	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14141	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14142	}
14143	}
14144	break;
14145	}
14146	}
14147	}
14148	#endif /* VBOX_STRICT */
14149	}
14150	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14151	{
14152	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14153	rcStrict = VINF_SUCCESS;
14154	}
14155	else
14156	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14157
14158	return rcStrict;
14159	}
14160
14161
14162	/**
14163	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14164	*/
14165	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14166	{
14167	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14168	/** @todo The guest has likely hit a contended spinlock. We might want to
14169	* poke a schedule different guest VCPU. */
14170	return VINF_EM_RAW_INTERRUPT;
14171	}
14172
14173
14174	/**
14175	* VM-exit handler for when the TPR value is lowered below the specified
14176	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14177	*/
14178	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14179	{
14180	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14181	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14182
14183	/*
14184	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14185	* We'll re-evaluate pending interrupts and inject them before the next VM
14186	* entry so we can just continue execution here.
14187	*/
14188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14189	return VINF_SUCCESS;
14190	}
14191
14192
14193	/**
14194	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14195	* VM-exit.
14196	*
14197	* @retval VINF_SUCCESS when guest execution can continue.
14198	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14199	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
14200	* interpreter.
14201	*/
14202	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14203	{
14204	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14205	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14206
14207	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14208	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14209	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14210	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14211	AssertRCReturn(rc, rc);
14212
14213	VBOXSTRICTRC rcStrict;
14214	PVM pVM = pVCpu->CTX_SUFF(pVM);
14215	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
14216	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14217	switch (uAccessType)
14218	{
14219	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14220	{
14221	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14222	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14223	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
14224	AssertMsg( rcStrict == VINF_SUCCESS
14225	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14226	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14227
14228	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14229	{
14230	case 0:
14231	{
14232	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14233	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14235	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14236
14237	/*
14238	* This is a kludge for handling switches back to real mode when we try to use
14239	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14240	* deal with special selector values, so we have to return to ring-3 and run
14241	* there till the selector values are V86 mode compatible.
14242	*
14243	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14244	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
14245	* at the end of this function.
14246	*/
14247	if ( rc == VINF_SUCCESS
14248	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
14249	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14250	&& (uOldCr0 & X86_CR0_PE)
14251	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
14252	{
14253	/** @todo check selectors rather than returning all the time. */
14254	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14255	rcStrict = VINF_EM_RESCHEDULE_REM;
14256	}
14257	break;
14258	}
14259
14260	case 2:
14261	{
14262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14263	/* Nothing to do here, CR2 it's not part of the VMCS. */
14264	break;
14265	}
14266
14267	case 3:
14268	{
14269	Assert( !pVM->hm.s.fNestedPaging
14270	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14271	\|\| pVCpu->hm.s.fUsingDebugLoop);
14272	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14273	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14274	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14275	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14276	break;
14277	}
14278
14279	case 4:
14280	{
14281	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14282	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14283	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14284	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14285	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14286	break;
14287	}
14288
14289	case 8:
14290	{
14291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14292	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14293	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14294	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14295	break;
14296	}
14297	default:
14298	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
14299	break;
14300	}
14301	break;
14302	}
14303
14304	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14305	{
14306	Assert( !pVM->hm.s.fNestedPaging
14307	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14308	\|\| pVCpu->hm.s.fUsingDebugLoop
14309	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
14310	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14311	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
14312	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14313
14314	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
14315	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
14316	AssertMsg( rcStrict == VINF_SUCCESS
14317	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14318	#ifdef VBOX_WITH_STATISTICS
14319	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14320	{
14321	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14322	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14323	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14324	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14325	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14326	}
14327	#endif
14328	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14329	VBOXSTRICTRC_VAL(rcStrict)));
14330	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14331	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14332	else
14333	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14334	break;
14335	}
14336
14337	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14338	{
14339	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14340	AssertMsg( rcStrict == VINF_SUCCESS
14341	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14342
14343	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14344	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14345	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14346	break;
14347	}
14348
14349	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14350	{
14351	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14352	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14353	AssertRCReturn(rc, rc);
14354	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14355	pVmxTransient->uGuestLinearAddr);
14356	AssertMsg( rcStrict == VINF_SUCCESS
14357	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14358	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14359
14360	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14361	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14362	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14363	break;
14364	}
14365
14366	default:
14367	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14368	VERR_VMX_UNEXPECTED_EXCEPTION);
14369	}
14370
14371	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14372	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14373	if (rcStrict == VINF_IEM_RAISED_XCPT)
14374	{
14375	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14376	rcStrict = VINF_SUCCESS;
14377	}
14378
14379	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14380	NOREF(pVM);
14381	return rcStrict;
14382	}
14383
14384
14385	/**
14386	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14387	* VM-exit.
14388	*/
14389	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14390	{
14391	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14392	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14393
14394	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14395	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14396	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14397	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14398	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14399	\| CPUMCTX_EXTRN_EFER);
14400	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14401	AssertRCReturn(rc, rc);
14402
14403	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14404	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14405	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14406	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14407	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14408	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14409	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14410	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14411
14412	/*
14413	* Update exit history to see if this exit can be optimized.
14414	*/
14415	VBOXSTRICTRC rcStrict;
14416	PCEMEXITREC pExitRec = NULL;
14417	if ( !fGstStepping
14418	&& !fDbgStepping)
14419	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14420	!fIOString
14421	? !fIOWrite
14422	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14423	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14424	: !fIOWrite
14425	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14426	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14427	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14428	if (!pExitRec)
14429	{
14430	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14431	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14432
14433	uint32_t const cbValue = s_aIOSizes[uIOSize];
14434	uint32_t const cbInstr = pVmxTransient->cbInstr;
14435	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14436	PVM pVM = pVCpu->CTX_SUFF(pVM);
14437	if (fIOString)
14438	{
14439	/*
14440	* INS/OUTS - I/O String instruction.
14441	*
14442	* Use instruction-information if available, otherwise fall back on
14443	* interpreting the instruction.
14444	*/
14445	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14446	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14447	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14448	if (fInsOutsInfo)
14449	{
14450	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14451	AssertRCReturn(rc2, rc2);
14452	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14453	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14454	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14455	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14456	if (fIOWrite)
14457	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14458	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14459	else
14460	{
14461	/*
14462	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14463	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14464	* See Intel Instruction spec. for "INS".
14465	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14466	*/
14467	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14468	}
14469	}
14470	else
14471	rcStrict = IEMExecOne(pVCpu);
14472
14473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14474	fUpdateRipAlready = true;
14475	}
14476	else
14477	{
14478	/*
14479	* IN/OUT - I/O instruction.
14480	*/
14481	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14482	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14483	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14484	if (fIOWrite)
14485	{
14486	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14487	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14488	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14489	&& !pCtx->eflags.Bits.u1TF)
14490	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14491	}
14492	else
14493	{
14494	uint32_t u32Result = 0;
14495	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14496	if (IOM_SUCCESS(rcStrict))
14497	{
14498	/* Save result of I/O IN instr. in AL/AX/EAX. */
14499	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14500	}
14501	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14502	&& !pCtx->eflags.Bits.u1TF)
14503	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14505	}
14506	}
14507
14508	if (IOM_SUCCESS(rcStrict))
14509	{
14510	if (!fUpdateRipAlready)
14511	{
14512	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14513	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14514	}
14515
14516	/*
14517	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14518	* while booting Fedora 17 64-bit guest.
14519	*
14520	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14521	*/
14522	if (fIOString)
14523	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14524
14525	/*
14526	* If any I/O breakpoints are armed, we need to check if one triggered
14527	* and take appropriate action.
14528	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14529	*/
14530	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14531	AssertRCReturn(rc, rc);
14532
14533	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14534	* execution engines about whether hyper BPs and such are pending. */
14535	uint32_t const uDr7 = pCtx->dr[7];
14536	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14537	&& X86_DR7_ANY_RW_IO(uDr7)
14538	&& (pCtx->cr4 & X86_CR4_DE))
14539	\|\| DBGFBpIsHwIoArmed(pVM)))
14540	{
14541	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14542
14543	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14544	VMMRZCallRing3Disable(pVCpu);
14545	HM_DISABLE_PREEMPT(pVCpu);
14546
14547	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14548
14549	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14550	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14551	{
14552	/* Raise #DB. */
14553	if (fIsGuestDbgActive)
14554	ASMSetDR6(pCtx->dr[6]);
14555	if (pCtx->dr[7] != uDr7)
14556	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14557
14558	hmR0VmxSetPendingXcptDB(pVCpu);
14559	}
14560	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14561	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14562	else if ( rcStrict2 != VINF_SUCCESS
14563	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14564	rcStrict = rcStrict2;
14565	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14566
14567	HM_RESTORE_PREEMPT();
14568	VMMRZCallRing3Enable(pVCpu);
14569	}
14570	}
14571
14572	#ifdef VBOX_STRICT
14573	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14574	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14575	Assert(!fIOWrite);
14576	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14577	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14578	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14579	Assert(fIOWrite);
14580	else
14581	{
14582	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14583	* statuses, that the VMM device and some others may return. See
14584	* IOM_SUCCESS() for guidance. */
14585	AssertMsg( RT_FAILURE(rcStrict)
14586	\|\| rcStrict == VINF_SUCCESS
14587	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14588	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14589	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14590	\|\| rcStrict == VINF_EM_RAW_TO_R3
14591	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14592	# endif
14593	}
14594	#endif
14595	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14596	}
14597	else
14598	{
14599	/*
14600	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14601	*/
14602	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14603	AssertRCReturn(rc2, rc2);
14604	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14605	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14606	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14607	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14608	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14609	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14610
14611	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14612	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14613
14614	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14615	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14616	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14617	}
14618	return rcStrict;
14619	}
14620
14621
14622	/**
14623	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14624	* VM-exit.
14625	*/
14626	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14627	{
14628	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14629
14630	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14631	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14632	AssertRCReturn(rc, rc);
14633	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14634	{
14635	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14636	AssertRCReturn(rc, rc);
14637	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14638	{
14639	uint32_t uErrCode;
14640	RTGCUINTPTR GCPtrFaultAddress;
14641	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14642	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14643	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14644	if (fErrorCodeValid)
14645	{
14646	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14647	AssertRCReturn(rc, rc);
14648	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14649	}
14650	else
14651	uErrCode = 0;
14652
14653	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14654	&& uVector == X86_XCPT_PF)
14655	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14656	else
14657	GCPtrFaultAddress = 0;
14658
14659	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14660	AssertRCReturn(rc, rc);
14661
14662	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14663	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14664
14665	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14667	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14668	}
14669	}
14670
14671	/* Fall back to the interpreter to emulate the task-switch. */
14672	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14673	return VERR_EM_INTERPRETER;
14674	}
14675
14676
14677	/**
14678	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14679	*/
14680	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14681	{
14682	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14683
14684	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14685	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14686	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14687	AssertRCReturn(rc, rc);
14688	return VINF_EM_DBG_STEPPED;
14689	}
14690
14691
14692	/**
14693	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14694	*/
14695	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14696	{
14697	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14698	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14699
14700	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14701	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14702	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14703	{
14704	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14705	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14706	{
14707	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14708	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14709	}
14710	}
14711	else
14712	{
14713	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14714	rcStrict1 = VINF_SUCCESS;
14715	return rcStrict1;
14716	}
14717
14718	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14719	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14720	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14721	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14722	AssertRCReturn(rc, rc);
14723
14724	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14725	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14726	VBOXSTRICTRC rcStrict2;
14727	switch (uAccessType)
14728	{
14729	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14730	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14731	{
14732	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14733	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14734	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14735
14736	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14737	GCPhys &= PAGE_BASE_GC_MASK;
14738	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14739	PVM pVM = pVCpu->CTX_SUFF(pVM);
14740	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14741	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14742
14743	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14744	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14745	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14746	CPUMCTX2CORE(pCtx), GCPhys);
14747	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14748	if ( rcStrict2 == VINF_SUCCESS
14749	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14750	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14751	{
14752	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14753	\| HM_CHANGED_GUEST_APIC_TPR);
14754	rcStrict2 = VINF_SUCCESS;
14755	}
14756	break;
14757	}
14758
14759	default:
14760	Log4Func(("uAccessType=%#x\n", uAccessType));
14761	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14762	break;
14763	}
14764
14765	if (rcStrict2 != VINF_SUCCESS)
14766	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14767	return rcStrict2;
14768	}
14769
14770
14771	/**
14772	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14773	* VM-exit.
14774	*/
14775	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14776	{
14777	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14778
14779	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14780	if (pVmxTransient->fWasGuestDebugStateActive)
14781	{
14782	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14783	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14784	}
14785
14786	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14787	if ( !pVCpu->hm.s.fSingleInstruction
14788	&& !pVmxTransient->fWasHyperDebugStateActive)
14789	{
14790	Assert(!DBGFIsStepping(pVCpu));
14791	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14792
14793	/* Don't intercept MOV DRx any more. */
14794	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14795	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14796	AssertRCReturn(rc, rc);
14797
14798	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14799	VMMRZCallRing3Disable(pVCpu);
14800	HM_DISABLE_PREEMPT(pVCpu);
14801
14802	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14803	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14804	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14805
14806	HM_RESTORE_PREEMPT();
14807	VMMRZCallRing3Enable(pVCpu);
14808
14809	#ifdef VBOX_WITH_STATISTICS
14810	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14811	AssertRCReturn(rc, rc);
14812	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14813	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14814	else
14815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14816	#endif
14817	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14818	return VINF_SUCCESS;
14819	}
14820
14821	/*
14822	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14823	* The EFER MSR is always up-to-date.
14824	* Update the segment registers and DR7 from the CPU.
14825	*/
14826	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14827	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14828	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14829	AssertRCReturn(rc, rc);
14830	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14831
14832	PVM pVM = pVCpu->CTX_SUFF(pVM);
14833	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14834	{
14835	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14836	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14837	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14838	if (RT_SUCCESS(rc))
14839	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14840	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14841	}
14842	else
14843	{
14844	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14845	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14846	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14848	}
14849
14850	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14851	if (RT_SUCCESS(rc))
14852	{
14853	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14854	AssertRCReturn(rc2, rc2);
14855	return VINF_SUCCESS;
14856	}
14857	return rc;
14858	}
14859
14860
14861	/**
14862	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14863	* Conditional VM-exit.
14864	*/
14865	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14866	{
14867	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14868	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14869
14870	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14871	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14872	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14873	{
14874	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14875	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14876	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14877	{
14878	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14879	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14880	}
14881	}
14882	else
14883	{
14884	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14885	rcStrict1 = VINF_SUCCESS;
14886	return rcStrict1;
14887	}
14888
14889	/*
14890	* Get sufficent state and update the exit history entry.
14891	*/
14892	RTGCPHYS GCPhys;
14893	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14894	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14895	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14896	AssertRCReturn(rc, rc);
14897
14898	VBOXSTRICTRC rcStrict;
14899	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14900	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14901	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14902	if (!pExitRec)
14903	{
14904	/*
14905	* If we succeed, resume guest execution.
14906	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14907	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14908	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14909	* weird case. See @bugref{6043}.
14910	*/
14911	PVM pVM = pVCpu->CTX_SUFF(pVM);
14912	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14913	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14914	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14915	if ( rcStrict == VINF_SUCCESS
14916	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14917	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14918	{
14919	/* Successfully handled MMIO operation. */
14920	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14921	\| HM_CHANGED_GUEST_APIC_TPR);
14922	rcStrict = VINF_SUCCESS;
14923	}
14924	}
14925	else
14926	{
14927	/*
14928	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14929	*/
14930	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14931	AssertRCReturn(rc2, rc2);
14932
14933	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14934	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14935
14936	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14937	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14938
14939	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14940	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14941	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14942	}
14943	return VBOXSTRICTRC_TODO(rcStrict);
14944	}
14945
14946
14947	/**
14948	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14949	* VM-exit.
14950	*/
14951	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14952	{
14953	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14954	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14955
14956	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14957	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14958	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14959	{
14960	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14961	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14962	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14963	}
14964	else
14965	{
14966	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14967	rcStrict1 = VINF_SUCCESS;
14968	return rcStrict1;
14969	}
14970
14971	RTGCPHYS GCPhys;
14972	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14973	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14974	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14975	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14976	AssertRCReturn(rc, rc);
14977
14978	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14979	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14980
14981	RTGCUINT uErrorCode = 0;
14982	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14983	uErrorCode \|= X86_TRAP_PF_ID;
14984	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14985	uErrorCode \|= X86_TRAP_PF_RW;
14986	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14987	uErrorCode \|= X86_TRAP_PF_P;
14988
14989	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14990
14991
14992	/* Handle the pagefault trap for the nested shadow table. */
14993	PVM pVM = pVCpu->CTX_SUFF(pVM);
14994	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14995
14996	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14997	pCtx->cs.Sel, pCtx->rip));
14998
14999	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15000	TRPMResetTrap(pVCpu);
15001
15002	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15003	if ( rcStrict2 == VINF_SUCCESS
15004	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15005	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
15006	{
15007	/* Successfully synced our nested page tables. */
15008	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15009	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15010	return VINF_SUCCESS;
15011	}
15012
15013	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15014	return rcStrict2;
15015	}
15016
15017	/** @} */
15018
15019	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15020	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
15021	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15022
15023	/**
15024	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
15025	*/
15026	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15027	{
15028	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15029	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15030
15031	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15032	AssertRCReturn(rc, rc);
15033
15034	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15035	{
15036	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15037	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15038
15039	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15040	* provides VM-exit instruction length. If this causes problem later,
15041	* disassemble the instruction like it's done on AMD-V. */
15042	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15043	AssertRCReturn(rc2, rc2);
15044	return rc;
15045	}
15046
15047	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15048	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15049	return rc;
15050	}
15051
15052
15053	/**
15054	* VM-exit exception handler for \#BP (Breakpoint exception).
15055	*/
15056	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15057	{
15058	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15059	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15060
15061	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15062	AssertRCReturn(rc, rc);
15063
15064	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15065	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15066	if (rc == VINF_EM_RAW_GUEST_TRAP)
15067	{
15068	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15069	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15070	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15071	AssertRCReturn(rc, rc);
15072
15073	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15074	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15075	}
15076
15077	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15078	return rc;
15079	}
15080
15081
15082	/**
15083	* VM-exit exception handler for \#AC (alignment check exception).
15084	*/
15085	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15086	{
15087	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15088
15089	/*
15090	* Re-inject it. We'll detect any nesting before getting here.
15091	*/
15092	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15093	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15094	AssertRCReturn(rc, rc);
15095	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15096
15097	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15098	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15099	return VINF_SUCCESS;
15100	}
15101
15102
15103	/**
15104	* VM-exit exception handler for \#DB (Debug exception).
15105	*/
15106	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15107	{
15108	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15109	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15110
15111	/*
15112	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
15113	* for processing.
15114	*/
15115	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15116
15117	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15118	uint64_t const uDR6 = X86_DR6_INIT_VAL
15119	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15120	\| X86_DR6_BD \| X86_DR6_BS));
15121
15122	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15123	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15124	Log6Func(("rc=%Rrc\n", rc));
15125	if (rc == VINF_EM_RAW_GUEST_TRAP)
15126	{
15127	/*
15128	* The exception was for the guest. Update DR6, DR7.GD and
15129	* IA32_DEBUGCTL.LBR before forwarding it.
15130	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15131	*/
15132	VMMRZCallRing3Disable(pVCpu);
15133	HM_DISABLE_PREEMPT(pVCpu);
15134
15135	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15136	pCtx->dr[6] \|= uDR6;
15137	if (CPUMIsGuestDebugStateActive(pVCpu))
15138	ASMSetDR6(pCtx->dr[6]);
15139
15140	HM_RESTORE_PREEMPT();
15141	VMMRZCallRing3Enable(pVCpu);
15142
15143	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15144	AssertRCReturn(rc, rc);
15145
15146	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15147	pCtx->dr[7] &= ~X86_DR7_GD;
15148
15149	/* Paranoia. */
15150	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15151	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15152
15153	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15154	AssertRCReturn(rc, rc);
15155
15156	/*
15157	* Raise #DB in the guest.
15158	*
15159	* It is important to reflect exactly what the VM-exit gave us (preserving the
15160	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15161	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15162	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15163	*
15164	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15165	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15166	*/
15167	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15168	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15169	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15170	AssertRCReturn(rc, rc);
15171	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15172	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15173	return VINF_SUCCESS;
15174	}
15175
15176	/*
15177	* Not a guest trap, must be a hypervisor related debug event then.
15178	* Update DR6 in case someone is interested in it.
15179	*/
15180	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15181	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15182	CPUMSetHyperDR6(pVCpu, uDR6);
15183
15184	return rc;
15185	}
15186
15187
15188	/**
15189	* Hacks its way around the lovely mesa driver's backdoor accesses.
15190	*
15191	* @sa hmR0SvmHandleMesaDrvGp.
15192	*/
15193	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15194	{
15195	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15196	RT_NOREF(pCtx);
15197
15198	/* For now we'll just skip the instruction. */
15199	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15200	}
15201
15202
15203	/**
15204	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15205	* backdoor logging w/o checking what it is running inside.
15206	*
15207	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15208	* backdoor port and magic numbers loaded in registers.
15209	*
15210	* @returns true if it is, false if it isn't.
15211	* @sa hmR0SvmIsMesaDrvGp.
15212	*/
15213	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15214	{
15215	/* 0xed: IN eAX,dx */
15216	uint8_t abInstr[1];
15217	if (pVmxTransient->cbInstr != sizeof(abInstr))
15218	return false;
15219
15220	/* Check that it is #GP(0). */
15221	if (pVmxTransient->uExitIntErrorCode != 0)
15222	return false;
15223
15224	/* Check magic and port. */
15225	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15226	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15227	if (pCtx->rax != UINT32_C(0x564d5868))
15228	return false;
15229	if (pCtx->dx != UINT32_C(0x5658))
15230	return false;
15231
15232	/* Flat ring-3 CS. */
15233	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15234	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15235	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15236	if (pCtx->cs.Attr.n.u2Dpl != 3)
15237	return false;
15238	if (pCtx->cs.u64Base != 0)
15239	return false;
15240
15241	/* Check opcode. */
15242	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15243	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15244	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15245	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15246	if (RT_FAILURE(rc))
15247	return false;
15248	if (abInstr[0] != 0xed)
15249	return false;
15250
15251	return true;
15252	}
15253
15254
15255	/**
15256	* VM-exit exception handler for \#GP (General-protection exception).
15257	*
15258	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15259	*/
15260	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15261	{
15262	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15263	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15264
15265	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15266	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15267	if (pVmcsInfo->RealMode.fRealOnV86Active)
15268	{ /* likely */ }
15269	else
15270	{
15271	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15272	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15273	#endif
15274	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15275	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15276	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15277	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15278	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15279	AssertRCReturn(rc, rc);
15280	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15281	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15282
15283	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15284	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15285	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15286	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15287	else
15288	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15289	return rc;
15290	}
15291
15292	Assert(CPUMIsGuestInRealModeEx(pCtx));
15293	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15294
15295	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15296	AssertRCReturn(rc, rc);
15297
15298	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15299	if (rcStrict == VINF_SUCCESS)
15300	{
15301	if (!CPUMIsGuestInRealModeEx(pCtx))
15302	{
15303	/*
15304	* The guest is no longer in real-mode, check if we can continue executing the
15305	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15306	*/
15307	pVmcsInfo->RealMode.fRealOnV86Active = false;
15308	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15309	{
15310	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15311	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15312	}
15313	else
15314	{
15315	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15316	rcStrict = VINF_EM_RESCHEDULE;
15317	}
15318	}
15319	else
15320	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15321	}
15322	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15323	{
15324	rcStrict = VINF_SUCCESS;
15325	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15326	}
15327	return VBOXSTRICTRC_VAL(rcStrict);
15328	}
15329
15330
15331	/**
15332	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15333	* the exception reported in the VMX transient structure back into the VM.
15334	*
15335	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15336	* up-to-date.
15337	*/
15338	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15339	{
15340	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15341	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15342	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15343	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15344	("uVector=%#x u32XcptBitmap=%#X32\n",
15345	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15346	NOREF(pVmcsInfo);
15347	#endif
15348
15349	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15350	hmR0VmxCheckExitDueToEventDelivery(). */
15351	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15352	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15353	AssertRCReturn(rc, rc);
15354	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15355
15356	#ifdef DEBUG_ramshankar
15357	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15358	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15359	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15360	#endif
15361
15362	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15363	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15364	return VINF_SUCCESS;
15365	}
15366
15367
15368	/**
15369	* VM-exit exception handler for \#PF (Page-fault exception).
15370	*/
15371	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15372	{
15373	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15374	PVM pVM = pVCpu->CTX_SUFF(pVM);
15375	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15376	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15377	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15378	AssertRCReturn(rc, rc);
15379
15380	if (!pVM->hm.s.fNestedPaging)
15381	{ /* likely */ }
15382	else
15383	{
15384	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15385	Assert(pVCpu->hm.s.fUsingDebugLoop);
15386	#endif
15387	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15388	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15389	{
15390	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15391	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15392	}
15393	else
15394	{
15395	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15396	hmR0VmxSetPendingXcptDF(pVCpu);
15397	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15398	}
15399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15400	return rc;
15401	}
15402
15403	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15404	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15405	if (pVmxTransient->fVectoringPF)
15406	{
15407	Assert(pVCpu->hm.s.Event.fPending);
15408	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15409	}
15410
15411	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15412	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15413	AssertRCReturn(rc, rc);
15414
15415	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15416	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15417
15418	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15419	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15420
15421	Log4Func(("#PF: rc=%Rrc\n", rc));
15422	if (rc == VINF_SUCCESS)
15423	{
15424	/*
15425	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15426	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15427	*/
15428	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15429	TRPMResetTrap(pVCpu);
15430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15431	return rc;
15432	}
15433
15434	if (rc == VINF_EM_RAW_GUEST_TRAP)
15435	{
15436	if (!pVmxTransient->fVectoringDoublePF)
15437	{
15438	/* It's a guest page fault and needs to be reflected to the guest. */
15439	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15440	TRPMResetTrap(pVCpu);
15441	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15442	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15443	uGstErrorCode, pVmxTransient->uExitQual);
15444	}
15445	else
15446	{
15447	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15448	TRPMResetTrap(pVCpu);
15449	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15450	hmR0VmxSetPendingXcptDF(pVCpu);
15451	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15452	}
15453
15454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15455	return VINF_SUCCESS;
15456	}
15457
15458	TRPMResetTrap(pVCpu);
15459	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15460	return rc;
15461	}
15462
15463	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15464	/** @name VMX instruction handlers.
15465	* @{
15466	*/
15467	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15468	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15469	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15470
15471	/**
15472	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15473	*/
15474	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15475	{
15476	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15477
15478	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15479	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15480	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15481	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15482	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15483	AssertRCReturn(rc, rc);
15484
15485	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15486
15487	VMXVEXITINFO ExitInfo;
15488	RT_ZERO(ExitInfo);
15489	ExitInfo.uReason = pVmxTransient->uExitReason;
15490	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15491	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15492	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15493	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15494
15495	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15496	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15497	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15498	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15499	{
15500	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15501	rcStrict = VINF_SUCCESS;
15502	}
15503	return rcStrict;
15504	}
15505
15506
15507	/**
15508	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15509	*/
15510	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15511	{
15512	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15513
15514	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15515	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15516	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15517	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15518	AssertRCReturn(rc, rc);
15519
15520	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15521
15522	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15523	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15524	{
15525	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15526	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15527	}
15528	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15529	return rcStrict;
15530	}
15531
15532
15533	/**
15534	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15535	*/
15536	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15537	{
15538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15539
15540	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15541	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15542	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15543	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15544	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15545	AssertRCReturn(rc, rc);
15546
15547	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15548
15549	VMXVEXITINFO ExitInfo;
15550	RT_ZERO(ExitInfo);
15551	ExitInfo.uReason = pVmxTransient->uExitReason;
15552	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15553	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15554	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15555	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15556
15557	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15558	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15559	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15560	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15561	{
15562	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15563	rcStrict = VINF_SUCCESS;
15564	}
15565	return rcStrict;
15566	}
15567
15568
15569	/**
15570	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15571	*/
15572	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15573	{
15574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15575
15576	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15577	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15578	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15579	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15580	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15581	AssertRCReturn(rc, rc);
15582
15583	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15584
15585	VMXVEXITINFO ExitInfo;
15586	RT_ZERO(ExitInfo);
15587	ExitInfo.uReason = pVmxTransient->uExitReason;
15588	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15589	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15590	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15591	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15592
15593	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15594	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15595	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15596	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15597	{
15598	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15599	rcStrict = VINF_SUCCESS;
15600	}
15601	return rcStrict;
15602	}
15603
15604
15605	/**
15606	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15607	*/
15608	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15609	{
15610	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15611
15612	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15613	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15614	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15615	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15616	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15617	AssertRCReturn(rc, rc);
15618
15619	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15620
15621	VMXVEXITINFO ExitInfo;
15622	RT_ZERO(ExitInfo);
15623	ExitInfo.uReason = pVmxTransient->uExitReason;
15624	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15625	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15626	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15627	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15628	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15629
15630	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15631	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15632	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15633	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15634	{
15635	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15636	rcStrict = VINF_SUCCESS;
15637	}
15638	return rcStrict;
15639	}
15640
15641
15642	/**
15643	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15644	*/
15645	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15646	{
15647	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15648
15649	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15650	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15651	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15652	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15653	AssertRCReturn(rc, rc);
15654
15655	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15656
15657	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15658	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15659	{
15660	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15661	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15662	}
15663	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15664	return rcStrict;
15665	}
15666
15667
15668	/**
15669	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15670	*/
15671	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15672	{
15673	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15674
15675	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15676	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15677	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15678	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15679	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15680	AssertRCReturn(rc, rc);
15681
15682	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15683
15684	VMXVEXITINFO ExitInfo;
15685	RT_ZERO(ExitInfo);
15686	ExitInfo.uReason = pVmxTransient->uExitReason;
15687	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15688	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15689	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15690	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15691	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15692
15693	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15694	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15695	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15696	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15697	{
15698	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15699	rcStrict = VINF_SUCCESS;
15700	}
15701	return rcStrict;
15702	}
15703
15704
15705	/**
15706	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15707	*/
15708	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15709	{
15710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15711
15712	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15713	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15714	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15715	AssertRCReturn(rc, rc);
15716
15717	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15718
15719	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15720	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15721	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15722	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15723	{
15724	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15725	rcStrict = VINF_SUCCESS;
15726	}
15727	return rcStrict;
15728	}
15729
15730
15731	/**
15732	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15733	*/
15734	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15735	{
15736	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15737
15738	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15739	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15740	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15741	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15742	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15743	AssertRCReturn(rc, rc);
15744
15745	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15746
15747	VMXVEXITINFO ExitInfo;
15748	RT_ZERO(ExitInfo);
15749	ExitInfo.uReason = pVmxTransient->uExitReason;
15750	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15751	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15752	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15753	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15754
15755	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15756	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15757	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15758	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15759	{
15760	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15761	rcStrict = VINF_SUCCESS;
15762	}
15763	return rcStrict;
15764	}
15765
15766
15767	/**
15768	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15769	*/
15770	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15771	{
15772	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15773
15774	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15775	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15776	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15777	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15778	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15779	AssertRCReturn(rc, rc);
15780
15781	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15782
15783	VMXVEXITINFO ExitInfo;
15784	RT_ZERO(ExitInfo);
15785	ExitInfo.uReason = pVmxTransient->uExitReason;
15786	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15787	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15788	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15789	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15790
15791	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15792	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15793	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15794	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15795	{
15796	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15797	rcStrict = VINF_SUCCESS;
15798	}
15799	return rcStrict;
15800	}
15801
15802	/** @} */
15803	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15804

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78640

Download in other formats: