HMVMXR0.cpp@ 78349

Last change on this file since 78349 was 78349, checked in by vboxsync, 6 years ago
HMVMXR0: Fix for machines with no EFER in the VMCS (was not saving host MSR value). Plus a few additionals comments and log statements.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 642.5 KB

Line
1	/* $Id: HMVMXR0.cpp 78349 2019-04-29 13:37:16Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	* @remarks Can be called with interrupts disabled.
1554	*/
1555	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1556	{
1557	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1558	Assert(pbMsrBitmap);
1559	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1560
1561	/*
1562	* MSR-bitmap Layout:
1563	* Byte index MSR range Interpreted as
1564	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1565	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1566	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1567	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1568	*
1569	* A bit corresponding to an MSR within the above range causes a VM-exit
1570	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1571	* the MSR range, it always cause a VM-exit.
1572	*
1573	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1574	*/
1575	uint16_t const offBitmapRead = 0;
1576	uint16_t const offBitmapWrite = 0x800;
1577	uint16_t offMsr;
1578	int32_t iBit;
1579	if (idMsr <= UINT32_C(0x00001fff))
1580	{
1581	offMsr = 0;
1582	iBit = idMsr;
1583	}
1584	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1585	{
1586	offMsr = 0x400;
1587	iBit = idMsr - UINT32_C(0xc0000000);
1588	}
1589	else
1590	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1591
1592	/*
1593	* Set the MSR read permission.
1594	*/
1595	uint16_t const offMsrRead = offBitmapRead + offMsr;
1596	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1597	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1598	{
1599	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1600	bool const fClear = !fIsNstGstVmcs ? true
1601	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1602	#else
1603	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1604	bool const fClear = true;
1605	#endif
1606	if (fClear)
1607	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1608	}
1609	else
1610	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1611
1612	/*
1613	* Set the MSR write permission.
1614	*/
1615	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1616	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1617	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1618	{
1619	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1620	bool const fClear = !fIsNstGstVmcs ? true
1621	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1622	#else
1623	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1624	bool const fClear = true;
1625	#endif
1626	if (fClear)
1627	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1628	}
1629	else
1630	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1631	}
1632
1633
1634	/**
1635	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1636	* area.
1637	*
1638	* @returns VBox status code.
1639	* @param pVCpu The cross context virtual CPU structure.
1640	* @param pVmcsInfo The VMCS info. object.
1641	* @param cMsrs The number of MSRs.
1642	*/
1643	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1644	{
1645	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1646	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1647	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1648	{
1649	/* Commit the MSR counts to the VMCS and update the cache. */
1650	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1651	{
1652	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1653	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1654	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1655	AssertRCReturn(rc, rc);
1656
1657	pVmcsInfo->cEntryMsrLoad = cMsrs;
1658	pVmcsInfo->cExitMsrStore = cMsrs;
1659	pVmcsInfo->cExitMsrLoad = cMsrs;
1660	}
1661	return VINF_SUCCESS;
1662	}
1663
1664	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1665	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1666	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1667	}
1668
1669
1670	/**
1671	* Adds a new (or updates the value of an existing) guest/host MSR
1672	* pair to be swapped during the world-switch as part of the
1673	* auto-load/store MSR area in the VMCS.
1674	*
1675	* @returns VBox status code.
1676	* @param pVCpu The cross context virtual CPU structure.
1677	* @param pVmxTransient The VMX-transient structure.
1678	* @param idMsr The MSR.
1679	* @param uGuestMsrValue Value of the guest MSR.
1680	* @param fSetReadWrite Whether to set the guest read/write access of this
1681	* MSR (thus not causing a VM-exit).
1682	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1683	* necessary.
1684	*/
1685	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1686	bool fSetReadWrite, bool fUpdateHostMsr)
1687	{
1688	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1689	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1690	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1691	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1692	uint32_t i;
1693
1694	/* Paranoia. */
1695	Assert(pGuestMsrLoad);
1696
1697	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1698
1699	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1700	for (i = 0; i < cMsrs; i++)
1701	{
1702	if (pGuestMsrLoad[i].u32Msr == idMsr)
1703	break;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1718
1719	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1720	fAdded = true;
1721	}
1722
1723	/* Update the MSR value for the newly added or already existing MSR. */
1724	pGuestMsrLoad[i].u32Msr = idMsr;
1725	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1726
1727	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1728	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1729	{
1730	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1731	pGuestMsrStore[i].u32Msr = idMsr;
1732	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1738	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1739	pHostMsr[i].u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if (fAdded)
1749	{
1750	if (fUpdateHostMsr)
1751	{
1752	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1753	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1754	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1755	}
1756	else
1757	{
1758	/* Someone else can do the work. */
1759	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1760	}
1761	}
1762	return VINF_SUCCESS;
1763	}
1764
1765
1766	/**
1767	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1768	* auto-load/store MSR area in the VMCS.
1769	*
1770	* @returns VBox status code.
1771	* @param pVCpu The cross context virtual CPU structure.
1772	* @param pVmxTransient The VMX-transient structure.
1773	* @param idMsr The MSR.
1774	*/
1775	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1776	{
1777	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1778	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1779	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1780	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1781
1782	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1783
1784	for (uint32_t i = 0; i < cMsrs; i++)
1785	{
1786	/* Find the MSR. */
1787	if (pGuestMsrLoad[i].u32Msr == idMsr)
1788	{
1789	/*
1790	* If it's the last MSR, we only need to reduce the MSR count.
1791	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1792	*/
1793	if (i < cMsrs - 1)
1794	{
1795	/* Remove it from the VM-entry MSR-load area. */
1796	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1797	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1798
1799	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1800	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1801	{
1802	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1803	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1804	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1805	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1806	}
1807
1808	/* Remove it from the VM-exit MSR-load area. */
1809	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1810	Assert(pHostMsr[i].u32Msr == idMsr);
1811	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1812	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1813	}
1814
1815	/* Reduce the count to reflect the removed MSR and bail. */
1816	--cMsrs;
1817	break;
1818	}
1819	}
1820
1821	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1822	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1823	{
1824	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1825	AssertRCReturn(rc, rc);
1826
1827	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1828	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1829	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1830
1831	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1832	return VINF_SUCCESS;
1833	}
1834
1835	return VERR_NOT_FOUND;
1836	}
1837
1838
1839	/**
1840	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1841	*
1842	* @returns @c true if found, @c false otherwise.
1843	* @param pVmcsInfo The VMCS info. object.
1844	* @param idMsr The MSR to find.
1845	*/
1846	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1847	{
1848	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1849	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1850	Assert(pMsrs);
1851	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1852	for (uint32_t i = 0; i < cMsrs; i++)
1853	{
1854	if (pMsrs[i].u32Msr == idMsr)
1855	return true;
1856	}
1857	return false;
1858	}
1859
1860
1861	/**
1862	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1863	*
1864	* @param pVCpu The cross context virtual CPU structure.
1865	* @param pVmcsInfo The VMCS info. object.
1866	*
1867	* @remarks No-long-jump zone!!!
1868	*/
1869	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1870	{
1871	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1872
1873	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1874	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1875	Assert(pHostMsrLoad);
1876	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1877	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1878	for (uint32_t i = 0; i < cMsrs; i++)
1879	{
1880	/*
1881	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1882	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1883	*/
1884	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1885	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1886	else
1887	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1888	}
1889	}
1890
1891
1892	/**
1893	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1894	* perform lazy restoration of the host MSRs while leaving VT-x.
1895	*
1896	* @param pVCpu The cross context virtual CPU structure.
1897	*
1898	* @remarks No-long-jump zone!!!
1899	*/
1900	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1901	{
1902	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1903
1904	/*
1905	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1906	*/
1907	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1908	{
1909	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1910	#if HC_ARCH_BITS == 64
1911	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1912	{
1913	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1914	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1915	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1916	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1917	}
1918	#endif
1919	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1920	}
1921	}
1922
1923
1924	/**
1925	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1926	* lazily while leaving VT-x.
1927	*
1928	* @returns true if it does, false otherwise.
1929	* @param pVCpu The cross context virtual CPU structure.
1930	* @param idMsr The MSR to check.
1931	*/
1932	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1933	{
1934	NOREF(pVCpu);
1935	#if HC_ARCH_BITS == 64
1936	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1937	{
1938	switch (idMsr)
1939	{
1940	case MSR_K8_LSTAR:
1941	case MSR_K6_STAR:
1942	case MSR_K8_SF_MASK:
1943	case MSR_K8_KERNEL_GS_BASE:
1944	return true;
1945	}
1946	}
1947	#else
1948	RT_NOREF(pVCpu, idMsr);
1949	#endif
1950	return false;
1951	}
1952
1953
1954	/**
1955	* Loads a set of guests MSRs to allow read/passthru to the guest.
1956	*
1957	* The name of this function is slightly confusing. This function does NOT
1958	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1959	* common prefix for functions dealing with "lazy restoration" of the shared
1960	* MSRs.
1961	*
1962	* @param pVCpu The cross context virtual CPU structure.
1963	*
1964	* @remarks No-long-jump zone!!!
1965	*/
1966	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1967	{
1968	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1969	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1970
1971	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1972	#if HC_ARCH_BITS == 64
1973	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1974	{
1975	/*
1976	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1977	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1978	* we can skip a few MSR writes.
1979	*
1980	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1981	* guest MSR values in the guest-CPU context might be different to what's currently
1982	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1983	* CPU, see @bugref{8728}.
1984	*/
1985	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1986	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1987	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1988	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1989	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1990	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1991	{
1992	#ifdef VBOX_STRICT
1993	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1994	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1995	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1996	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1997	#endif
1998	}
1999	else
2000	{
2001	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2002	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2003	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2004	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2005	}
2006	}
2007	#endif
2008	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2009	}
2010
2011
2012	/**
2013	* Performs lazy restoration of the set of host MSRs if they were previously
2014	* loaded with guest MSR values.
2015	*
2016	* @param pVCpu The cross context virtual CPU structure.
2017	*
2018	* @remarks No-long-jump zone!!!
2019	* @remarks The guest MSRs should have been saved back into the guest-CPU
2020	* context by hmR0VmxImportGuestState()!!!
2021	*/
2022	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2023	{
2024	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2025	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2026
2027	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2028	{
2029	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2030	#if HC_ARCH_BITS == 64
2031	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2032	{
2033	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2034	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2035	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2036	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2037	}
2038	#endif
2039	}
2040	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2041	}
2042
2043
2044	/**
2045	* Verifies that our cached values of the VMCS fields are all consistent with
2046	* what's actually present in the VMCS.
2047	*
2048	* @returns VBox status code.
2049	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2050	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2051	* VMCS content. HMCPU error-field is
2052	* updated, see VMX_VCI_XXX.
2053	* @param pVCpu The cross context virtual CPU structure.
2054	* @param pVmcsInfo The VMCS info. object.
2055	*/
2056	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2057	{
2058	uint32_t u32Val;
2059	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2060	AssertRCReturn(rc, rc);
2061	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2062	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2063	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2064	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2065
2066	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2067	AssertRCReturn(rc, rc);
2068	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2069	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2070	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2071	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2072
2073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2074	AssertRCReturn(rc, rc);
2075	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2076	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2077	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2078	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2079
2080	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2081	AssertRCReturn(rc, rc);
2082	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2083	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2084	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2085	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2086
2087	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2088	{
2089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2090	AssertRCReturn(rc, rc);
2091	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2092	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2093	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2094	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2095	}
2096
2097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2098	AssertRCReturn(rc, rc);
2099	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2100	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2101	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2102	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2103
2104	uint64_t u64Val;
2105	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2106	AssertRCReturn(rc, rc);
2107	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2108	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2109	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2110	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2111
2112	return VINF_SUCCESS;
2113	}
2114
2115
2116	#ifdef VBOX_STRICT
2117	/**
2118	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2119	*
2120	* @param pVCpu The cross context virtual CPU structure.
2121	* @param pVmcsInfo The VMCS info. object.
2122	*/
2123	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2124	{
2125	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2126
2127	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2128	{
2129	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2130	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2131	uint64_t uVmcsEferMsrVmcs;
2132	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2133	AssertRC(rc);
2134
2135	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2136	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2137	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2138	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2139	}
2140	}
2141
2142
2143	/**
2144	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2145	* VMCS are correct.
2146	*
2147	* @param pVCpu The cross context virtual CPU structure.
2148	* @param pVmcsInfo The VMCS info. object.
2149	*/
2150	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2151	{
2152	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2153
2154	/* Read the various MSR-area counts from the VMCS. */
2155	uint32_t cEntryLoadMsrs;
2156	uint32_t cExitStoreMsrs;
2157	uint32_t cExitLoadMsrs;
2158	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2159	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2160	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2161
2162	/* Verify all the MSR counts are the same. */
2163	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2164	Assert(cExitStoreMsrs == cExitLoadMsrs);
2165	uint32_t const cMsrs = cExitLoadMsrs;
2166
2167	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2168	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2169
2170	/* Verify the MSR counts are within the allocated page size. */
2171	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2172
2173	/* Verify the relevant contents of the MSR areas match. */
2174	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2175	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2176	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2177	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2178	for (uint32_t i = 0; i < cMsrs; i++)
2179	{
2180	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2181	if (fSeparateExitMsrStorePage)
2182	{
2183	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2184	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2185	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2186	}
2187
2188	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2189	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2190	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2191
2192	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2193	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2194	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2195	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2196
2197	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2198	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2199	{
2200	uint32_t const fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2201	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2202	{
2203	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2204	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2205	}
2206	else
2207	{
2208	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2209	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2210	}
2211	}
2212
2213	/* Move to the next MSR. */
2214	pHostMsrLoad++;
2215	pGuestMsrLoad++;
2216	pGuestMsrStore++;
2217	}
2218	}
2219	#endif /* VBOX_STRICT */
2220
2221
2222	/**
2223	* Flushes the TLB using EPT.
2224	*
2225	* @returns VBox status code.
2226	* @param pVCpu The cross context virtual CPU structure of the calling
2227	* EMT. Can be NULL depending on @a enmTlbFlush.
2228	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2229	* enmTlbFlush.
2230	* @param enmTlbFlush Type of flush.
2231	*
2232	* @remarks Caller is responsible for making sure this function is called only
2233	* when NestedPaging is supported and providing @a enmTlbFlush that is
2234	* supported by the CPU.
2235	* @remarks Can be called with interrupts disabled.
2236	*/
2237	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2238	{
2239	uint64_t au64Descriptor[2];
2240	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2241	au64Descriptor[0] = 0;
2242	else
2243	{
2244	Assert(pVCpu);
2245	Assert(pVmcsInfo);
2246	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2247	}
2248	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2249
2250	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2251	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2252
2253	if ( RT_SUCCESS(rc)
2254	&& pVCpu)
2255	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2256	}
2257
2258
2259	/**
2260	* Flushes the TLB using VPID.
2261	*
2262	* @returns VBox status code.
2263	* @param pVCpu The cross context virtual CPU structure of the calling
2264	* EMT. Can be NULL depending on @a enmTlbFlush.
2265	* @param enmTlbFlush Type of flush.
2266	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2267	* on @a enmTlbFlush).
2268	*
2269	* @remarks Can be called with interrupts disabled.
2270	*/
2271	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2272	{
2273	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2274
2275	uint64_t au64Descriptor[2];
2276	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2277	{
2278	au64Descriptor[0] = 0;
2279	au64Descriptor[1] = 0;
2280	}
2281	else
2282	{
2283	AssertPtr(pVCpu);
2284	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2285	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2286	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2287	au64Descriptor[1] = GCPtr;
2288	}
2289
2290	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2291	AssertMsg(rc == VINF_SUCCESS,
2292	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2293
2294	if ( RT_SUCCESS(rc)
2295	&& pVCpu)
2296	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2297	NOREF(rc);
2298	}
2299
2300
2301	/**
2302	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2303	* otherwise there is nothing really to invalidate.
2304	*
2305	* @returns VBox status code.
2306	* @param pVCpu The cross context virtual CPU structure.
2307	* @param GCVirt Guest virtual address of the page to invalidate.
2308	*/
2309	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2310	{
2311	AssertPtr(pVCpu);
2312	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2313
2314	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2315	{
2316	/*
2317	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2318	* the EPT case. See @bugref{6043} and @bugref{6177}.
2319	*
2320	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2321	* as this function maybe called in a loop with individual addresses.
2322	*/
2323	PVM pVM = pVCpu->CTX_SUFF(pVM);
2324	if (pVM->hm.s.vmx.fVpid)
2325	{
2326	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2327
2328	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2329	/*
2330	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2331	* where executing INVVPID outside 64-bit mode does not flush translations of
2332	* 64-bit linear addresses, see @bugref{6208#c72}.
2333	*/
2334	if (RT_HI_U32(GCVirt))
2335	fVpidFlush = false;
2336	#endif
2337
2338	if (fVpidFlush)
2339	{
2340	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2341	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2342	}
2343	else
2344	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2345	}
2346	else if (pVM->hm.s.fNestedPaging)
2347	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2348	}
2349
2350	return VINF_SUCCESS;
2351	}
2352
2353
2354	/**
2355	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2356	* case where neither EPT nor VPID is supported by the CPU.
2357	*
2358	* @param pHostCpu The HM physical-CPU structure.
2359	* @param pVCpu The cross context virtual CPU structure.
2360	*
2361	* @remarks Called with interrupts disabled.
2362	*/
2363	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2364	{
2365	AssertPtr(pVCpu);
2366	AssertPtr(pHostCpu);
2367
2368	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2369
2370	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2371	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2372	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2373	pVCpu->hm.s.fForceTLBFlush = false;
2374	return;
2375	}
2376
2377
2378	/**
2379	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2380	*
2381	* @param pHostCpu The HM physical-CPU structure.
2382	* @param pVCpu The cross context virtual CPU structure.
2383	* @param pVmcsInfo The VMCS info. object.
2384	*
2385	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2386	* nomenclature. The reason is, to avoid confusion in compare statements
2387	* since the host-CPU copies are named "ASID".
2388	*
2389	* @remarks Called with interrupts disabled.
2390	*/
2391	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2392	{
2393	#ifdef VBOX_WITH_STATISTICS
2394	bool fTlbFlushed = false;
2395	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2396	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2397	if (!fTlbFlushed) \
2398	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2399	} while (0)
2400	#else
2401	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2402	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2403	#endif
2404
2405	AssertPtr(pVCpu);
2406	AssertPtr(pHostCpu);
2407	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2408
2409	PVM pVM = pVCpu->CTX_SUFF(pVM);
2410	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2411	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2412	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2413
2414	/*
2415	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2416	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2417	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2418	* cannot reuse the current ASID anymore.
2419	*/
2420	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2421	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2422	{
2423	++pHostCpu->uCurrentAsid;
2424	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2425	{
2426	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2427	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2428	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2429	}
2430
2431	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2432	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2433	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2434
2435	/*
2436	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2437	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2438	*/
2439	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2440	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2441	HMVMX_SET_TAGGED_TLB_FLUSHED();
2442	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2443	}
2444	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2445	{
2446	/*
2447	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2448	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2449	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2450	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2451	* mappings, see @bugref{6568}.
2452	*
2453	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2454	*/
2455	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2456	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2457	HMVMX_SET_TAGGED_TLB_FLUSHED();
2458	}
2459
2460	pVCpu->hm.s.fForceTLBFlush = false;
2461	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2462
2463	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2464	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2465	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2466	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2467	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2468	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2469	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2470	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2471	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2472
2473	/* Update VMCS with the VPID. */
2474	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2475	AssertRC(rc);
2476
2477	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2478	}
2479
2480
2481	/**
2482	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2483	*
2484	* @param pHostCpu The HM physical-CPU structure.
2485	* @param pVCpu The cross context virtual CPU structure.
2486	* @param pVmcsInfo The VMCS info. object.
2487	*
2488	* @remarks Called with interrupts disabled.
2489	*/
2490	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2491	{
2492	AssertPtr(pVCpu);
2493	AssertPtr(pHostCpu);
2494	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2495	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2496	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2497
2498	/*
2499	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2500	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2501	*/
2502	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2503	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2504	{
2505	pVCpu->hm.s.fForceTLBFlush = true;
2506	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2507	}
2508
2509	/* Check for explicit TLB flushes. */
2510	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2511	{
2512	pVCpu->hm.s.fForceTLBFlush = true;
2513	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2514	}
2515
2516	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2517	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2518
2519	if (pVCpu->hm.s.fForceTLBFlush)
2520	{
2521	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2522	pVCpu->hm.s.fForceTLBFlush = false;
2523	}
2524	}
2525
2526
2527	/**
2528	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2529	*
2530	* @param pHostCpu The HM physical-CPU structure.
2531	* @param pVCpu The cross context virtual CPU structure.
2532	*
2533	* @remarks Called with interrupts disabled.
2534	*/
2535	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2536	{
2537	AssertPtr(pVCpu);
2538	AssertPtr(pHostCpu);
2539	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2540	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2541	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2542
2543	/*
2544	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2545	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2546	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2547	* cannot reuse the current ASID anymore.
2548	*/
2549	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2550	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2551	{
2552	pVCpu->hm.s.fForceTLBFlush = true;
2553	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2554	}
2555
2556	/* Check for explicit TLB flushes. */
2557	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2558	{
2559	/*
2560	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2561	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2562	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2563	* include fExplicitFlush's too) - an obscure corner case.
2564	*/
2565	pVCpu->hm.s.fForceTLBFlush = true;
2566	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2567	}
2568
2569	PVM pVM = pVCpu->CTX_SUFF(pVM);
2570	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2571	if (pVCpu->hm.s.fForceTLBFlush)
2572	{
2573	++pHostCpu->uCurrentAsid;
2574	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2575	{
2576	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2577	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2578	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2579	}
2580
2581	pVCpu->hm.s.fForceTLBFlush = false;
2582	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2583	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2584	if (pHostCpu->fFlushAsidBeforeUse)
2585	{
2586	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2587	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2588	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2589	{
2590	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2591	pHostCpu->fFlushAsidBeforeUse = false;
2592	}
2593	else
2594	{
2595	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2596	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2597	}
2598	}
2599	}
2600
2601	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2602	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2603	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2604	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2605	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2606	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2607	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2608
2609	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2610	AssertRC(rc);
2611	}
2612
2613
2614	/**
2615	* Flushes the guest TLB entry based on CPU capabilities.
2616	*
2617	* @param pHostCpu The HM physical-CPU structure.
2618	* @param pVCpu The cross context virtual CPU structure.
2619	* @param pVmcsInfo The VMCS info. object.
2620	*
2621	* @remarks Called with interrupts disabled.
2622	*/
2623	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2624	{
2625	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2626	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2627	#endif
2628	PVM pVM = pVCpu->CTX_SUFF(pVM);
2629	switch (pVM->hm.s.vmx.enmTlbFlushType)
2630	{
2631	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2632	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2633	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2634	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2635	default:
2636	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2637	break;
2638	}
2639	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2640	}
2641
2642
2643	/**
2644	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2645	* TLB entries from the host TLB before VM-entry.
2646	*
2647	* @returns VBox status code.
2648	* @param pVM The cross context VM structure.
2649	*/
2650	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2651	{
2652	/*
2653	* Determine optimal flush type for nested paging.
2654	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2655	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2656	*/
2657	if (pVM->hm.s.fNestedPaging)
2658	{
2659	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2660	{
2661	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2662	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2663	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2664	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2665	else
2666	{
2667	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2668	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2669	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2670	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2671	}
2672
2673	/* Make sure the write-back cacheable memory type for EPT is supported. */
2674	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2675	{
2676	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2677	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2678	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2679	}
2680
2681	/* EPT requires a page-walk length of 4. */
2682	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2683	{
2684	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2685	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2686	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2687	}
2688	}
2689	else
2690	{
2691	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2692	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2693	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2694	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2695	}
2696	}
2697
2698	/*
2699	* Determine optimal flush type for VPID.
2700	*/
2701	if (pVM->hm.s.vmx.fVpid)
2702	{
2703	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2704	{
2705	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2706	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2707	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2708	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2709	else
2710	{
2711	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2712	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2713	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2714	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2715	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2716	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2717	pVM->hm.s.vmx.fVpid = false;
2718	}
2719	}
2720	else
2721	{
2722	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2723	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2724	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2725	pVM->hm.s.vmx.fVpid = false;
2726	}
2727	}
2728
2729	/*
2730	* Setup the handler for flushing tagged-TLBs.
2731	*/
2732	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2733	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2734	else if (pVM->hm.s.fNestedPaging)
2735	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2736	else if (pVM->hm.s.vmx.fVpid)
2737	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2738	else
2739	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2740	return VINF_SUCCESS;
2741	}
2742
2743
2744	/**
2745	* Sets up the virtual-APIC page address for the VMCS.
2746	*
2747	* @returns VBox status code.
2748	* @param pVCpu The cross context virtual CPU structure.
2749	* @param pVmcsInfo The VMCS info. object.
2750	*/
2751	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2752	{
2753	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2754	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2755	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2756	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2757	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2758	}
2759
2760
2761	/**
2762	* Sets up the MSR-bitmap address for the VMCS.
2763	*
2764	* @returns VBox status code.
2765	* @param pVCpu The cross context virtual CPU structure.
2766	* @param pVmcsInfo The VMCS info. object.
2767	*/
2768	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2769	{
2770	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2771	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2772	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2773	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2774	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2775	}
2776
2777
2778	/**
2779	* Sets up the APIC-access page address for the VMCS.
2780	*
2781	* @returns VBox status code.
2782	* @param pVCpu The cross context virtual CPU structure.
2783	*/
2784	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2785	{
2786	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2787	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2788	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2789	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2790	}
2791
2792
2793	/**
2794	* Sets up the VMCS link pointer for the VMCS.
2795	*
2796	* @returns VBox status code.
2797	* @param pVCpu The cross context virtual CPU structure.
2798	* @param pVmcsInfo The VMCS info. object.
2799	*/
2800	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2801	{
2802	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2803	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2804	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2805	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2806	}
2807
2808
2809	/**
2810	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2811	* in the VMCS.
2812	*
2813	* @returns VBox status code.
2814	* @param pVCpu The cross context virtual CPU structure.
2815	* @param pVmcsInfo The VMCS info. object.
2816	*/
2817	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2818	{
2819	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2820
2821	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2822	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2823	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2824
2825	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2826	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2827	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2828
2829	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2830	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2831	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2832
2833	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2834	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2835	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2836	AssertRCReturn(rc, rc);
2837	return VINF_SUCCESS;
2838	}
2839
2840
2841	/**
2842	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2843	*
2844	* @param pVCpu The cross context virtual CPU structure.
2845	* @param pVmcsInfo The VMCS info. object.
2846	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2847	*/
2848	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2849	{
2850	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2851
2852	/*
2853	* The guest can access the following MSRs (read, write) without causing
2854	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2855	*/
2856	PVM pVM = pVCpu->CTX_SUFF(pVM);
2857	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2858	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2859	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2860	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2861	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2862
2863	/*
2864	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2865	* associated with then. We never need to intercept access (writes need to be
2866	* executed without causing a VM-exit, reads will #GP fault anyway).
2867	*
2868	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2869	* read/write them. We swap the the guest/host MSR value using the
2870	* auto-load/store MSR area.
2871	*/
2872	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2873	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2874	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2875	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2876	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2877	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2878
2879	#if HC_ARCH_BITS == 64
2880	/*
2881	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2882	* required for 64-bit guests.
2883	*/
2884	if (pVM->hm.s.fAllow64BitGuests)
2885	{
2886	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2887	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2888	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2889	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2890	}
2891	#endif
2892
2893	/*
2894	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2895	*/
2896	#ifdef VBOX_STRICT
2897	Assert(pVmcsInfo->pvMsrBitmap);
2898	uint32_t const fMsrpmEfer = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2899	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2900	#endif
2901	}
2902
2903
2904	/**
2905	* Sets up pin-based VM-execution controls in the VMCS.
2906	*
2907	* @returns VBox status code.
2908	* @param pVCpu The cross context virtual CPU structure.
2909	* @param pVmcsInfo The VMCS info. object.
2910	*/
2911	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2912	{
2913	PVM pVM = pVCpu->CTX_SUFF(pVM);
2914	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2915	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2916
2917	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2918	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2919
2920	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2921	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2922
2923	/* Enable the VMX-preemption timer. */
2924	if (pVM->hm.s.vmx.fUsePreemptTimer)
2925	{
2926	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2927	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2928	}
2929
2930	#if 0
2931	/* Enable posted-interrupt processing. */
2932	if (pVM->hm.s.fPostedIntrs)
2933	{
2934	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2935	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2936	fVal \|= VMX_PIN_CTL_POSTED_INT;
2937	}
2938	#endif
2939
2940	if ((fVal & fZap) != fVal)
2941	{
2942	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2943	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2944	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2945	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2946	}
2947
2948	/* Commit it to the VMCS and update our cache. */
2949	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2950	AssertRCReturn(rc, rc);
2951	pVmcsInfo->u32PinCtls = fVal;
2952
2953	return VINF_SUCCESS;
2954	}
2955
2956
2957	/**
2958	* Sets up secondary processor-based VM-execution controls in the VMCS.
2959	*
2960	* @returns VBox status code.
2961	* @param pVCpu The cross context virtual CPU structure.
2962	* @param pVmcsInfo The VMCS info. object.
2963	*/
2964	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2965	{
2966	PVM pVM = pVCpu->CTX_SUFF(pVM);
2967	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2968	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2969
2970	/* WBINVD causes a VM-exit. */
2971	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2972	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2973
2974	/* Enable EPT (aka nested-paging). */
2975	if (pVM->hm.s.fNestedPaging)
2976	fVal \|= VMX_PROC_CTLS2_EPT;
2977
2978	/* Enable the INVPCID instruction if supported by the hardware and we expose
2979	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2980	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2981	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2982	fVal \|= VMX_PROC_CTLS2_INVPCID;
2983
2984	/* Enable VPID. */
2985	if (pVM->hm.s.vmx.fVpid)
2986	fVal \|= VMX_PROC_CTLS2_VPID;
2987
2988	/* Enable unrestricted guest execution. */
2989	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2990	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2991
2992	#if 0
2993	if (pVM->hm.s.fVirtApicRegs)
2994	{
2995	/* Enable APIC-register virtualization. */
2996	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2997	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2998
2999	/* Enable virtual-interrupt delivery. */
3000	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3001	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3002	}
3003	#endif
3004
3005	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3006	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3007	* done dynamically. */
3008	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3009	{
3010	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3011	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3012	AssertRCReturn(rc, rc);
3013	}
3014
3015	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3016	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3017	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3018	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3019	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3020
3021	/* Enable Pause-Loop exiting. */
3022	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3023	&& pVM->hm.s.vmx.cPleGapTicks
3024	&& pVM->hm.s.vmx.cPleWindowTicks)
3025	{
3026	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3027
3028	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3029	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3030	AssertRCReturn(rc, rc);
3031	}
3032
3033	if ((fVal & fZap) != fVal)
3034	{
3035	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3036	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3037	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3038	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3039	}
3040
3041	/* Commit it to the VMCS and update our cache. */
3042	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3043	AssertRCReturn(rc, rc);
3044	pVmcsInfo->u32ProcCtls2 = fVal;
3045
3046	return VINF_SUCCESS;
3047	}
3048
3049
3050	/**
3051	* Sets up processor-based VM-execution controls in the VMCS.
3052	*
3053	* @returns VBox status code.
3054	* @param pVCpu The cross context virtual CPU structure.
3055	* @param pVmcsInfo The VMCS info. object.
3056	*/
3057	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3058	{
3059	PVM pVM = pVCpu->CTX_SUFF(pVM);
3060
3061	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3062	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3063
3064	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3065	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3066	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3067	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3068	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3069	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3070	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3071
3072	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3073	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3074	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3075	{
3076	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3077	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3078	}
3079
3080	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3081	if (!pVM->hm.s.fNestedPaging)
3082	{
3083	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3084	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3085	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3086	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3087	}
3088
3089	/* Use TPR shadowing if supported by the CPU. */
3090	if ( PDMHasApic(pVM)
3091	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3092	{
3093	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3094	/* CR8 writes cause a VM-exit based on TPR threshold. */
3095	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3096	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3097	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3098	AssertRCReturn(rc, rc);
3099	}
3100	else
3101	{
3102	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3103	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3104	if (pVM->hm.s.fAllow64BitGuests)
3105	{
3106	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3107	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3108	}
3109	}
3110
3111	/* Use MSR-bitmaps if supported by the CPU. */
3112	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3113	{
3114	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3115	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3116	AssertRCReturn(rc, rc);
3117	}
3118
3119	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3120	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3121	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3122
3123	if ((fVal & fZap) != fVal)
3124	{
3125	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3126	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3127	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3128	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3129	}
3130
3131	/* Commit it to the VMCS and update our cache. */
3132	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3133	AssertRCReturn(rc, rc);
3134	pVmcsInfo->u32ProcCtls = fVal;
3135
3136	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3137	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3138	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3139
3140	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3141	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3142	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3143
3144	/* Sanity check, should not really happen. */
3145	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3146	{ /* likely */ }
3147	else
3148	{
3149	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3150	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3151	}
3152
3153	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3154	return VINF_SUCCESS;
3155	}
3156
3157
3158	/**
3159	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3160	* Processor-based VM-execution) control fields in the VMCS.
3161	*
3162	* @returns VBox status code.
3163	* @param pVCpu The cross context virtual CPU structure.
3164	* @param pVmcsInfo The VMCS info. object.
3165	*/
3166	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3167	{
3168	/* Set the auto-load/store MSR area addresses in the VMCS. */
3169	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3170	if (RT_SUCCESS(rc))
3171	{
3172	/* Set the VMCS link pointer in the VMCS. */
3173	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3174	if (RT_SUCCESS(rc))
3175	{
3176	/* Set the CR0/CR4 guest/host mask. */
3177	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3178	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3179	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3180	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3181	if (RT_SUCCESS(rc))
3182	{
3183	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3184	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3185	return VINF_SUCCESS;
3186	}
3187	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3188	}
3189	else
3190	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3191	}
3192	else
3193	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3194	return rc;
3195	}
3196
3197
3198	/**
3199	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3200	*
3201	* We shall setup those exception intercepts that don't change during the
3202	* lifetime of the VM here. The rest are done dynamically while loading the
3203	* guest state.
3204	*
3205	* @returns VBox status code.
3206	* @param pVCpu The cross context virtual CPU structure.
3207	* @param pVmcsInfo The VMCS info. object.
3208	*/
3209	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3210	{
3211	/*
3212	* The following exceptions are always intercepted:
3213	*
3214	* #AC - To prevent the guest from hanging the CPU.
3215	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3216	* recursive #DBs can cause a CPU hang.
3217	* #PF - To sync our shadow page tables when nested-paging is not used.
3218	*/
3219	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3220	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3221	\| RT_BIT(X86_XCPT_DB)
3222	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3223
3224	/* Commit it to the VMCS. */
3225	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3226	AssertRCReturn(rc, rc);
3227
3228	/* Update our cache of the exception bitmap. */
3229	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3230	return VINF_SUCCESS;
3231	}
3232
3233
3234	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3235	/**
3236	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3237	*
3238	* @returns VBox status code.
3239	* @param pVCpu The cross context virtual CPU structure.
3240	* @param pVmcsInfo The VMCS info. object.
3241	*/
3242	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3243	{
3244	PVM pVM = pVCpu->CTX_SUFF(pVM);
3245	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3246	if (RT_SUCCESS(rc))
3247	{
3248	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3249	if (RT_SUCCESS(rc))
3250	{
3251	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3252	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3253	if (RT_SUCCESS(rc))
3254	{
3255	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3256	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3257	if (RT_SUCCESS(rc))
3258	return VINF_SUCCESS;
3259
3260	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3261	}
3262	else
3263	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3264	}
3265	else
3266	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3267	}
3268	else
3269	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3270
3271	return rc;
3272	}
3273	#endif
3274
3275
3276	/**
3277	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3278	* VMX.
3279	*
3280	* @returns VBox status code.
3281	* @param pVCpu The cross context virtual CPU structure.
3282	* @param pVmcsInfo The VMCS info. object.
3283	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3284	*/
3285	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3286	{
3287	Assert(pVmcsInfo);
3288	Assert(pVmcsInfo->pvVmcs);
3289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3290
3291	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3292	PVM pVM = pVCpu->CTX_SUFF(pVM);
3293	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3294	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3295
3296	LogFlowFunc(("\n"));
3297
3298	/*
3299	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3300	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3301	*/
3302	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3303	if (RT_SUCCESS(rc))
3304	{
3305	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3306	if (RT_SUCCESS(rc))
3307	{
3308	if (!fIsNstGstVmcs)
3309	{
3310	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3311	if (RT_SUCCESS(rc))
3312	{
3313	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3314	if (RT_SUCCESS(rc))
3315	{
3316	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3317	if (RT_SUCCESS(rc))
3318	{
3319	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3320	if (RT_SUCCESS(rc))
3321	{ /* likely */ }
3322	else
3323	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3324	}
3325	else
3326	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3327	}
3328	else
3329	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3330	}
3331	else
3332	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3333	}
3334	else
3335	{
3336	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3337	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3338	if (RT_SUCCESS(rc))
3339	{ /* likely */ }
3340	else
3341	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3342	#else
3343	AssertFailed();
3344	#endif
3345	}
3346	}
3347	else
3348	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3349	}
3350	else
3351	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3352
3353	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3354	if (RT_SUCCESS(rc))
3355	{
3356	rc = hmR0VmxClearVmcs(pVmcsInfo);
3357	if (RT_SUCCESS(rc))
3358	{ /* likely */ }
3359	else
3360	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3361	}
3362
3363	/*
3364	* Update the last-error record both for failures and success, so we
3365	* can propagate the status code back to ring-3 for diagnostics.
3366	*/
3367	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3368	NOREF(pszVmcs);
3369	return rc;
3370	}
3371
3372
3373	/**
3374	* Does global VT-x initialization (called during module initialization).
3375	*
3376	* @returns VBox status code.
3377	*/
3378	VMMR0DECL(int) VMXR0GlobalInit(void)
3379	{
3380	#ifdef HMVMX_USE_FUNCTION_TABLE
3381	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3382	# ifdef VBOX_STRICT
3383	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3384	Assert(g_apfnVMExitHandlers[i]);
3385	# endif
3386	#endif
3387	return VINF_SUCCESS;
3388	}
3389
3390
3391	/**
3392	* Does global VT-x termination (called during module termination).
3393	*/
3394	VMMR0DECL(void) VMXR0GlobalTerm()
3395	{
3396	/* Nothing to do currently. */
3397	}
3398
3399
3400	/**
3401	* Sets up and activates VT-x on the current CPU.
3402	*
3403	* @returns VBox status code.
3404	* @param pHostCpu The HM physical-CPU structure.
3405	* @param pVM The cross context VM structure. Can be
3406	* NULL after a host resume operation.
3407	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3408	* fEnabledByHost is @c true).
3409	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3410	* @a fEnabledByHost is @c true).
3411	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3412	* enable VT-x on the host.
3413	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3414	*/
3415	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3416	PCSUPHWVIRTMSRS pHwvirtMsrs)
3417	{
3418	Assert(pHostCpu);
3419	Assert(pHwvirtMsrs);
3420	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3421
3422	/* Enable VT-x if it's not already enabled by the host. */
3423	if (!fEnabledByHost)
3424	{
3425	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3426	if (RT_FAILURE(rc))
3427	return rc;
3428	}
3429
3430	/*
3431	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3432	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3433	* invalidated when flushing by VPID.
3434	*/
3435	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3436	{
3437	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3438	pHostCpu->fFlushAsidBeforeUse = false;
3439	}
3440	else
3441	pHostCpu->fFlushAsidBeforeUse = true;
3442
3443	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3444	++pHostCpu->cTlbFlushes;
3445
3446	return VINF_SUCCESS;
3447	}
3448
3449
3450	/**
3451	* Deactivates VT-x on the current CPU.
3452	*
3453	* @returns VBox status code.
3454	* @param pvCpuPage Pointer to the VMXON region.
3455	* @param HCPhysCpuPage Physical address of the VMXON region.
3456	*
3457	* @remarks This function should never be called when SUPR0EnableVTx() or
3458	* similar was used to enable VT-x on the host.
3459	*/
3460	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3461	{
3462	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3463
3464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3465	return hmR0VmxLeaveRootMode();
3466	}
3467
3468
3469	/**
3470	* Does per-VM VT-x initialization.
3471	*
3472	* @returns VBox status code.
3473	* @param pVM The cross context VM structure.
3474	*/
3475	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3476	{
3477	LogFlowFunc(("pVM=%p\n", pVM));
3478
3479	int rc = hmR0VmxStructsAlloc(pVM);
3480	if (RT_FAILURE(rc))
3481	{
3482	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3483	return rc;
3484	}
3485
3486	return VINF_SUCCESS;
3487	}
3488
3489
3490	/**
3491	* Does per-VM VT-x termination.
3492	*
3493	* @returns VBox status code.
3494	* @param pVM The cross context VM structure.
3495	*/
3496	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3497	{
3498	LogFlowFunc(("pVM=%p\n", pVM));
3499
3500	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3501	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3502	{
3503	Assert(pVM->hm.s.vmx.pvScratch);
3504	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3505	}
3506	#endif
3507	hmR0VmxStructsFree(pVM);
3508	return VINF_SUCCESS;
3509	}
3510
3511
3512	/**
3513	* Sets up the VM for execution using hardware-assisted VMX.
3514	* This function is only called once per-VM during initialization.
3515	*
3516	* @returns VBox status code.
3517	* @param pVM The cross context VM structure.
3518	*/
3519	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3520	{
3521	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3522	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3523
3524	LogFlowFunc(("pVM=%p\n", pVM));
3525
3526	/*
3527	* At least verify if VMX is enabled, since we can't check if we're in
3528	* VMX root mode or not without causing a #GP.
3529	*/
3530	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3531	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3532	{ /* likely */ }
3533	else
3534	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3535
3536	/*
3537	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3538	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3539	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3540	*/
3541	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3542	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3543	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3544	{
3545	LogRelFunc(("Invalid real-on-v86 state.\n"));
3546	return VERR_INTERNAL_ERROR;
3547	}
3548
3549	/* Initialize these always, see hmR3InitFinalizeR0().*/
3550	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3551	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3552
3553	/* Setup the tagged-TLB flush handlers. */
3554	int rc = hmR0VmxSetupTaggedTlb(pVM);
3555	if (RT_FAILURE(rc))
3556	{
3557	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3558	return rc;
3559	}
3560
3561	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3562	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3563	#if HC_ARCH_BITS == 64
3564	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3565	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3566	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3567	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3568	#endif
3569
3570	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3571	{
3572	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3573	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3574
3575	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3576	if (RT_SUCCESS(rc))
3577	{
3578	#if HC_ARCH_BITS == 32
3579	hmR0VmxInitVmcsReadCache(pVCpu);
3580	#endif
3581	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3582	if (pVM->cpum.ro.GuestFeatures.fVmx)
3583	{
3584	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3585	if (RT_SUCCESS(rc))
3586	{ /* likely */ }
3587	else
3588	{
3589	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3590	return rc;
3591	}
3592	}
3593	#endif
3594	}
3595	else
3596	{
3597	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3598	return rc;
3599	}
3600	}
3601
3602	return VINF_SUCCESS;
3603	}
3604
3605
3606	#if HC_ARCH_BITS == 32
3607	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3608	/**
3609	* Check if guest state allows safe use of 32-bit switcher again.
3610	*
3611	* Segment bases and protected mode structures must be 32-bit addressable
3612	* because the 32-bit switcher will ignore high dword when writing these VMCS
3613	* fields. See @bugref{8432} for details.
3614	*
3615	* @returns true if safe, false if must continue to use the 64-bit switcher.
3616	* @param pCtx Pointer to the guest-CPU context.
3617	*
3618	* @remarks No-long-jump zone!!!
3619	*/
3620	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3621	{
3622	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3623	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3624	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3626	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3627	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3628	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3629	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3630	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3631	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3632
3633	/* All good, bases are 32-bit. */
3634	return true;
3635	}
3636	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3637
3638	# ifdef VBOX_STRICT
3639	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3640	{
3641	switch (idxField)
3642	{
3643	case VMX_VMCS_GUEST_RIP:
3644	case VMX_VMCS_GUEST_RSP:
3645	case VMX_VMCS_GUEST_SYSENTER_EIP:
3646	case VMX_VMCS_GUEST_SYSENTER_ESP:
3647	case VMX_VMCS_GUEST_GDTR_BASE:
3648	case VMX_VMCS_GUEST_IDTR_BASE:
3649	case VMX_VMCS_GUEST_CS_BASE:
3650	case VMX_VMCS_GUEST_DS_BASE:
3651	case VMX_VMCS_GUEST_ES_BASE:
3652	case VMX_VMCS_GUEST_FS_BASE:
3653	case VMX_VMCS_GUEST_GS_BASE:
3654	case VMX_VMCS_GUEST_SS_BASE:
3655	case VMX_VMCS_GUEST_LDTR_BASE:
3656	case VMX_VMCS_GUEST_TR_BASE:
3657	case VMX_VMCS_GUEST_CR3:
3658	return true;
3659	}
3660	return false;
3661	}
3662
3663	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3664	{
3665	switch (idxField)
3666	{
3667	/* Read-only fields. */
3668	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3669	return true;
3670	}
3671	/* Remaining readable fields should also be writable. */
3672	return hmR0VmxIsValidWriteField(idxField);
3673	}
3674	# endif /* VBOX_STRICT */
3675
3676
3677	/**
3678	* Executes the specified handler in 64-bit mode.
3679	*
3680	* @returns VBox status code (no informational status codes).
3681	* @param pVCpu The cross context virtual CPU structure.
3682	* @param enmOp The operation to perform.
3683	* @param cParams Number of parameters.
3684	* @param paParam Array of 32-bit parameters.
3685	*/
3686	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3687	{
3688	PVM pVM = pVCpu->CTX_SUFF(pVM);
3689	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3690	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3691	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3692	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3693
3694	#ifdef VBOX_STRICT
3695	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3696	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3697
3698	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3699	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3700	#endif
3701
3702	/* Disable interrupts. */
3703	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3704
3705	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3706	RTCPUID idHostCpu = RTMpCpuId();
3707	CPUMR0SetLApic(pVCpu, idHostCpu);
3708	#endif
3709
3710	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3711
3712	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3713	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3714
3715	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3716	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3717	hmR0VmxClearVmcs(pVmcsInfo);
3718
3719	/* Leave VMX root mode and disable VMX. */
3720	VMXDisable();
3721	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3722
3723	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3724	CPUMSetHyperEIP(pVCpu, enmOp);
3725	for (int i = (int)cParams - 1; i >= 0; i--)
3726	CPUMPushHyper(pVCpu, paParam[i]);
3727
3728	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3729
3730	/* Call the switcher. */
3731	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3732	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3733
3734	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3735	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3736
3737	/* Re-enter VMX root mode. */
3738	int rc2 = VMXEnable(HCPhysCpuPage);
3739	if (RT_FAILURE(rc2))
3740	{
3741	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3742	ASMSetFlags(fOldEFlags);
3743	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3744	return rc2;
3745	}
3746
3747	/* Restore the VMCS as the current VMCS. */
3748	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3749	AssertRC(rc2);
3750	Assert(!(ASMGetFlags() & X86_EFL_IF));
3751	ASMSetFlags(fOldEFlags);
3752	return rc;
3753	}
3754
3755
3756	/**
3757	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3758	* supporting 64-bit guests.
3759	*
3760	* @returns VBox status code.
3761	* @param fResume Whether to VMLAUNCH or VMRESUME.
3762	* @param pCtx Pointer to the guest-CPU context.
3763	* @param pCache Pointer to the VMCS batch cache.
3764	* @param pVM The cross context VM structure.
3765	* @param pVCpu The cross context virtual CPU structure.
3766	*/
3767	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3768	{
3769	NOREF(fResume);
3770
3771	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3772	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3773	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3774
3775	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3776	pCache->uPos = 1;
3777	pCache->interPD = PGMGetInterPaeCR3(pVM);
3778	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3779	#endif
3780
3781	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3782	pCache->TestIn.HCPhysCpuPage = 0;
3783	pCache->TestIn.HCPhysVmcs = 0;
3784	pCache->TestIn.pCache = 0;
3785	pCache->TestOut.HCPhysVmcs = 0;
3786	pCache->TestOut.pCache = 0;
3787	pCache->TestOut.pCtx = 0;
3788	pCache->TestOut.eflags = 0;
3789	#else
3790	NOREF(pCache);
3791	#endif
3792
3793	uint32_t aParam[10];
3794	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3795	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3796	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3797	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3798	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3799	aParam[5] = 0;
3800	aParam[6] = VM_RC_ADDR(pVM, pVM);
3801	aParam[7] = 0;
3802	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3803	aParam[9] = 0;
3804
3805	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3806	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3807	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3808	#endif
3809	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3810
3811	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3812	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3813	Assert(pCtx->dr[4] == 10);
3814	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3815	#endif
3816
3817	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3818	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3819	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3820	pVmcsInfo->HCPhysVmcs));
3821	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3822	pCache->TestOut.HCPhysVmcs));
3823	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3824	pCache->TestOut.pCache));
3825	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3826	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3827	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3828	pCache->TestOut.pCtx));
3829	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3830	#endif
3831	NOREF(pCtx);
3832	return rc;
3833	}
3834	#endif
3835
3836
3837	/**
3838	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3839	* the VMCS.
3840	*
3841	* @returns VBox status code.
3842	*/
3843	static int hmR0VmxExportHostControlRegs(void)
3844	{
3845	RTCCUINTREG uReg = ASMGetCR0();
3846	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3847	AssertRCReturn(rc, rc);
3848
3849	uReg = ASMGetCR3();
3850	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3851	AssertRCReturn(rc, rc);
3852
3853	uReg = ASMGetCR4();
3854	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3855	AssertRCReturn(rc, rc);
3856	return rc;
3857	}
3858
3859
3860	/**
3861	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3862	* the host-state area in the VMCS.
3863	*
3864	* @returns VBox status code.
3865	* @param pVCpu The cross context virtual CPU structure.
3866	*/
3867	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3868	{
3869	#if HC_ARCH_BITS == 64
3870	/**
3871	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3872	* requirements. See hmR0VmxExportHostSegmentRegs().
3873	*/
3874	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3875	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3876	{ \
3877	bool fValidSelector = true; \
3878	if ((selValue) & X86_SEL_LDT) \
3879	{ \
3880	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3881	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3882	} \
3883	if (fValidSelector) \
3884	{ \
3885	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3886	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3887	} \
3888	(selValue) = 0; \
3889	}
3890
3891	/*
3892	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3893	* will be messed up. We should -not- save the messed up state without restoring
3894	* the original host-state, see @bugref{7240}.
3895	*
3896	* This apparently can happen (most likely the FPU changes), deal with it rather than
3897	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3898	*/
3899	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3900	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3901	{
3902	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3903	pVCpu->idCpu));
3904	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3905	}
3906	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3907	#else
3908	RT_NOREF(pVCpu);
3909	#endif
3910
3911	/*
3912	* Host DS, ES, FS and GS segment registers.
3913	*/
3914	#if HC_ARCH_BITS == 64
3915	RTSEL uSelDS = ASMGetDS();
3916	RTSEL uSelES = ASMGetES();
3917	RTSEL uSelFS = ASMGetFS();
3918	RTSEL uSelGS = ASMGetGS();
3919	#else
3920	RTSEL uSelDS = 0;
3921	RTSEL uSelES = 0;
3922	RTSEL uSelFS = 0;
3923	RTSEL uSelGS = 0;
3924	#endif
3925
3926	/*
3927	* Host CS and SS segment registers.
3928	*/
3929	RTSEL uSelCS = ASMGetCS();
3930	RTSEL uSelSS = ASMGetSS();
3931
3932	/*
3933	* Host TR segment register.
3934	*/
3935	RTSEL uSelTR = ASMGetTR();
3936
3937	#if HC_ARCH_BITS == 64
3938	/*
3939	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3940	* gain VM-entry and restore them before we get preempted.
3941	*
3942	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3943	*/
3944	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3945	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3946	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3947	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3948	# undef VMXLOCAL_ADJUST_HOST_SEG
3949	#endif
3950
3951	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3952	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3953	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3954	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3955	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3956	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3957	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3958	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3959	Assert(uSelCS);
3960	Assert(uSelTR);
3961
3962	/* Write these host selector fields into the host-state area in the VMCS. */
3963	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3964	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3965	#if HC_ARCH_BITS == 64
3966	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3967	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3968	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3969	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3970	#else
3971	NOREF(uSelDS);
3972	NOREF(uSelES);
3973	NOREF(uSelFS);
3974	NOREF(uSelGS);
3975	#endif
3976	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3977	AssertRCReturn(rc, rc);
3978
3979	/*
3980	* Host GDTR and IDTR.
3981	*/
3982	RTGDTR Gdtr;
3983	RTIDTR Idtr;
3984	RT_ZERO(Gdtr);
3985	RT_ZERO(Idtr);
3986	ASMGetGDTR(&Gdtr);
3987	ASMGetIDTR(&Idtr);
3988	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3989	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3990	AssertRCReturn(rc, rc);
3991
3992	#if HC_ARCH_BITS == 64
3993	/*
3994	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3995	* them to the maximum limit (0xffff) on every VM-exit.
3996	*/
3997	if (Gdtr.cbGdt != 0xffff)
3998	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3999
4000	/*
4001	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4002	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4003	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4004	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4005	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4006	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4007	* at 0xffff on hosts where we are sure it won't cause trouble.
4008	*/
4009	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4010	if (Idtr.cbIdt < 0x0fff)
4011	# else
4012	if (Idtr.cbIdt != 0xffff)
4013	# endif
4014	{
4015	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4016	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4017	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4018	}
4019	#endif
4020
4021	/*
4022	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4023	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4024	* RPL should be too in most cases.
4025	*/
4026	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4027	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4028
4029	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4030	#if HC_ARCH_BITS == 64
4031	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4032
4033	/*
4034	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4035	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4036	* restoration if the host has something else. Task switching is not supported in 64-bit
4037	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4038	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4039	*
4040	* [1] See Intel spec. 3.5 "System Descriptor Types".
4041	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4042	*/
4043	PVM pVM = pVCpu->CTX_SUFF(pVM);
4044	Assert(pDesc->System.u4Type == 11);
4045	if ( pDesc->System.u16LimitLow != 0x67
4046	\|\| pDesc->System.u4LimitHigh)
4047	{
4048	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4049	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4050	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4051	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4052	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4053	}
4054
4055	/*
4056	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4057	*/
4058	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4059	{
4060	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4061	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4062	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4063	{
4064	/* The GDT is read-only but the writable GDT is available. */
4065	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4066	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4067	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4068	AssertRCReturn(rc, rc);
4069	}
4070	}
4071	#else
4072	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4073	#endif
4074	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4075	AssertRCReturn(rc, rc);
4076
4077	/*
4078	* Host FS base and GS base.
4079	*/
4080	#if HC_ARCH_BITS == 64
4081	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4082	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4083	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4084	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4085	AssertRCReturn(rc, rc);
4086
4087	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4088	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4089	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4090	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4091	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4092	#endif
4093	return VINF_SUCCESS;
4094	}
4095
4096
4097	/**
4098	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4099	* host-state area of the VMCS.
4100	*
4101	* These MSRs will be automatically restored on the host after every successful
4102	* VM-exit.
4103	*
4104	* @returns VBox status code.
4105	* @param pVCpu The cross context virtual CPU structure.
4106	*
4107	* @remarks No-long-jump zone!!!
4108	*/
4109	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4110	{
4111	AssertPtr(pVCpu);
4112
4113	/*
4114	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4115	* rather than swapping them on every VM-entry.
4116	*/
4117	hmR0VmxLazySaveHostMsrs(pVCpu);
4118
4119	/*
4120	* Host Sysenter MSRs.
4121	*/
4122	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4123	#if HC_ARCH_BITS == 32
4124	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4125	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4126	#else
4127	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4128	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4129	#endif
4130	AssertRCReturn(rc, rc);
4131
4132	/*
4133	* Host EFER MSR.
4134	*
4135	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4136	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4137	*/
4138	PVM pVM = pVCpu->CTX_SUFF(pVM);
4139	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4140	{
4141	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4142	AssertRCReturn(rc, rc);
4143	}
4144
4145	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4146	* hmR0VmxExportGuestEntryExitCtls(). */
4147
4148	return VINF_SUCCESS;
4149	}
4150
4151
4152	/**
4153	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4154	*
4155	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4156	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4157	*
4158	* @returns true if we need to load guest EFER, false otherwise.
4159	* @param pVCpu The cross context virtual CPU structure.
4160	*
4161	* @remarks Requires EFER, CR4.
4162	* @remarks No-long-jump zone!!!
4163	*/
4164	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4165	{
4166	#ifdef HMVMX_ALWAYS_SWAP_EFER
4167	RT_NOREF(pVCpu);
4168	return true;
4169	#else
4170	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4171	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4172	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4173	if (CPUMIsGuestInLongModeEx(pCtx))
4174	return false;
4175	#endif
4176
4177	PVM pVM = pVCpu->CTX_SUFF(pVM);
4178	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4179	uint64_t const u64GuestEfer = pCtx->msrEFER;
4180
4181	/*
4182	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4183	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4184	*/
4185	if ( CPUMIsGuestInLongModeEx(pCtx)
4186	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4187	return true;
4188
4189	/*
4190	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4191	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4192	*
4193	* See Intel spec. 4.5 "IA-32e Paging".
4194	* See Intel spec. 4.1.1 "Three Paging Modes".
4195	*
4196	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4197	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4198	*/
4199	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4200	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4201	if ( (pCtx->cr4 & X86_CR4_PAE)
4202	&& (pCtx->cr0 & X86_CR0_PG)
4203	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4204	{
4205	/* Assert that host is NX capable. */
4206	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4207	return true;
4208	}
4209
4210	return false;
4211	#endif
4212	}
4213
4214	/**
4215	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4216	* VMCS.
4217	*
4218	* This is typically required when the guest changes paging mode.
4219	*
4220	* @returns VBox status code.
4221	* @param pVCpu The cross context virtual CPU structure.
4222	* @param pVmxTransient The VMX-transient structure.
4223	*
4224	* @remarks Requires EFER.
4225	* @remarks No-long-jump zone!!!
4226	*/
4227	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4228	{
4229	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4230	{
4231	PVM pVM = pVCpu->CTX_SUFF(pVM);
4232	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4233
4234	/*
4235	* VM-entry controls.
4236	*/
4237	{
4238	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4239	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4240
4241	/*
4242	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4243	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4244	*
4245	* For nested-guests, this is a mandatory VM-entry control. It's also
4246	* required because we do not want to leak host bits to the nested-guest.
4247	*/
4248	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4249
4250	/*
4251	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4252	*
4253	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4254	* required to get the nested-guest working with hardware-assisted VMX execution.
4255	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4256	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4257	* here rather than while merging the guest VMCS controls.
4258	*/
4259	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4260	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4261	else
4262	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4263
4264	/*
4265	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4266	*
4267	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4268	* regardless of whether the nested-guest VMCS specifies it because we are free to
4269	* load whatever MSRs we require and we do not need to modify the guest visible copy
4270	* of the VM-entry MSR load area.
4271	*/
4272	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4273	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4274	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4275	else
4276	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4277
4278	/*
4279	* The following should -not- be set (since we're not in SMM mode):
4280	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4281	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4282	*/
4283
4284	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4285	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4286
4287	if ((fVal & fZap) == fVal)
4288	{ /* likely */ }
4289	else
4290	{
4291	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4292	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4293	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4294	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4295	}
4296
4297	/* Commit it to the VMCS. */
4298	if (pVmcsInfo->u32EntryCtls != fVal)
4299	{
4300	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4301	AssertRCReturn(rc, rc);
4302	pVmcsInfo->u32EntryCtls = fVal;
4303	}
4304	}
4305
4306	/*
4307	* VM-exit controls.
4308	*/
4309	{
4310	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4311	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4312
4313	/*
4314	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4315	* supported the 1-setting of this bit.
4316	*
4317	* For nested-guests, we set the "save debug controls" as the converse
4318	* "load debug controls" is mandatory for nested-guests anyway.
4319	*/
4320	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4321
4322	/*
4323	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4324	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4325	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4326	* hmR0VmxExportHostMsrs().
4327	*
4328	* For nested-guests, we always set this bit as we do not support 32-bit
4329	* hosts.
4330	*/
4331	#if HC_ARCH_BITS == 64
4332	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4333	#else
4334	Assert(!pVmxTransient->fIsNestedGuest);
4335	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4336	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4337	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4338	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4339	{
4340	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4341	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4342	}
4343	else
4344	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4345	#endif
4346
4347	/*
4348	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4349	*
4350	* For nested-guests, we should use the "save IA32_EFER" control if we also
4351	* used the "load IA32_EFER" control while exporting VM-entry controls.
4352	*/
4353	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4354	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4355	{
4356	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4357	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4358	}
4359
4360	/*
4361	* Enable saving of the VMX-preemption timer value on VM-exit.
4362	* For nested-guests, currently not exposed/used.
4363	*/
4364	if ( pVM->hm.s.vmx.fUsePreemptTimer
4365	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4366	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4367
4368	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4369	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4370
4371	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4372	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4373	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4374
4375	if ((fVal & fZap) == fVal)
4376	{ /* likely */ }
4377	else
4378	{
4379	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4380	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4381	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4382	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4383	}
4384
4385	/* Commit it to the VMCS. */
4386	if (pVmcsInfo->u32ExitCtls != fVal)
4387	{
4388	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4389	AssertRCReturn(rc, rc);
4390	pVmcsInfo->u32ExitCtls = fVal;
4391	}
4392	}
4393
4394	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4395	}
4396	return VINF_SUCCESS;
4397	}
4398
4399
4400	/**
4401	* Sets the TPR threshold in the VMCS.
4402	*
4403	* @returns VBox status code.
4404	* @param pVCpu The cross context virtual CPU structure.
4405	* @param pVmcsInfo The VMCS info. object.
4406	* @param u32TprThreshold The TPR threshold (task-priority class only).
4407	*/
4408	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4409	{
4410	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4411	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4412	RT_NOREF2(pVCpu, pVmcsInfo);
4413	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4414	}
4415
4416
4417	/**
4418	* Exports the guest APIC TPR state into the VMCS.
4419	*
4420	* @returns VBox status code.
4421	* @param pVCpu The cross context virtual CPU structure.
4422	* @param pVmxTransient The VMX-transient structure.
4423	*
4424	* @remarks No-long-jump zone!!!
4425	*/
4426	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4427	{
4428	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4429	{
4430	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4431
4432	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4433	if (!pVmxTransient->fIsNestedGuest)
4434	{
4435	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4436	&& APICIsEnabled(pVCpu))
4437	{
4438	/*
4439	* Setup TPR shadowing.
4440	*/
4441	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4442	{
4443	bool fPendingIntr = false;
4444	uint8_t u8Tpr = 0;
4445	uint8_t u8PendingIntr = 0;
4446	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4447	AssertRCReturn(rc, rc);
4448
4449	/*
4450	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4451	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4452	* priority of the pending interrupt so we can deliver the interrupt. If there
4453	* are no interrupts pending, set threshold to 0 to not cause any
4454	* TPR-below-threshold VM-exits.
4455	*/
4456	Assert(pVmcsInfo->pbVirtApic);
4457	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4458	uint32_t u32TprThreshold = 0;
4459	if (fPendingIntr)
4460	{
4461	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4462	(which is the Task-Priority Class). */
4463	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4464	const uint8_t u8TprPriority = u8Tpr >> 4;
4465	if (u8PendingPriority <= u8TprPriority)
4466	u32TprThreshold = u8PendingPriority;
4467	}
4468
4469	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4470	AssertRCReturn(rc, rc);
4471	}
4472	}
4473	}
4474	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4475	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4476	}
4477	return VINF_SUCCESS;
4478	}
4479
4480
4481	/**
4482	* Gets the guest interruptibility-state.
4483	*
4484	* @returns Guest's interruptibility-state.
4485	* @param pVCpu The cross context virtual CPU structure.
4486	* @param pVmcsInfo The VMCS info. object.
4487	*
4488	* @remarks No-long-jump zone!!!
4489	*/
4490	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4491	{
4492	/*
4493	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4494	*/
4495	uint32_t fIntrState = 0;
4496	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4497	{
4498	/* If inhibition is active, RIP and RFLAGS should've been updated
4499	(i.e. read previously from the VMCS or from ring-3). */
4500	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4501	#ifdef VBOX_STRICT
4502	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4503	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4504	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4505	#endif
4506	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4507	{
4508	if (pCtx->eflags.Bits.u1IF)
4509	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4510	else
4511	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4512	}
4513	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4514	{
4515	/*
4516	* We can clear the inhibit force flag as even if we go back to the recompiler
4517	* without executing guest code in VT-x, the flag's condition to be cleared is
4518	* met and thus the cleared state is correct.
4519	*/
4520	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4521	}
4522	}
4523
4524	/*
4525	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4526	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4527	* setting this would block host-NMIs and IRET will not clear the blocking.
4528	*
4529	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4530	*
4531	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4532	*/
4533	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4534	&& CPUMIsGuestNmiBlocking(pVCpu))
4535	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4536
4537	return fIntrState;
4538	}
4539
4540
4541	/**
4542	* Exports the exception intercepts required for guest execution in the VMCS.
4543	*
4544	* @returns VBox status code.
4545	* @param pVCpu The cross context virtual CPU structure.
4546	* @param pVmxTransient The VMX-transient structure.
4547	*
4548	* @remarks No-long-jump zone!!!
4549	*/
4550	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4551	{
4552	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4553	{
4554	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4555	if ( !pVmxTransient->fIsNestedGuest
4556	&& pVCpu->hm.s.fGIMTrapXcptUD)
4557	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4558	else
4559	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4560
4561	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4562	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4563	}
4564	return VINF_SUCCESS;
4565	}
4566
4567
4568	/**
4569	* Exports the guest's RIP into the guest-state area in the VMCS.
4570	*
4571	* @returns VBox status code.
4572	* @param pVCpu The cross context virtual CPU structure.
4573	*
4574	* @remarks No-long-jump zone!!!
4575	*/
4576	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4577	{
4578	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4579	{
4580	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4581
4582	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4583	AssertRCReturn(rc, rc);
4584
4585	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4586	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4587	}
4588	return VINF_SUCCESS;
4589	}
4590
4591
4592	/**
4593	* Exports the guest's RSP into the guest-state area in the VMCS.
4594	*
4595	* @returns VBox status code.
4596	* @param pVCpu The cross context virtual CPU structure.
4597	*
4598	* @remarks No-long-jump zone!!!
4599	*/
4600	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4601	{
4602	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4603	{
4604	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4605
4606	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4607	AssertRCReturn(rc, rc);
4608
4609	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4610	}
4611	return VINF_SUCCESS;
4612	}
4613
4614
4615	/**
4616	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4617	*
4618	* @returns VBox status code.
4619	* @param pVCpu The cross context virtual CPU structure.
4620	* @param pVmxTransient The VMX-transient structure.
4621	*
4622	* @remarks No-long-jump zone!!!
4623	*/
4624	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4625	{
4626	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4627	{
4628	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4629
4630	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4631	Let us assert it as such and use 32-bit VMWRITE. */
4632	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4633	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4634	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4635	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4636
4637	/*
4638	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4639	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4640	* can run the real-mode guest code under Virtual 8086 mode.
4641	*/
4642	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4643	if (pVmcsInfo->RealMode.fRealOnV86Active)
4644	{
4645	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4646	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4647	Assert(!pVmxTransient->fIsNestedGuest);
4648	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4649	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4650	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4651	}
4652
4653	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4654	AssertRCReturn(rc, rc);
4655
4656	/*
4657	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4658	*
4659	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4660	* through the hypervisor debugger using EFLAGS.TF.
4661	*/
4662	if ( !pVmxTransient->fIsNestedGuest
4663	&& !pVCpu->hm.s.fSingleInstruction
4664	&& fEFlags.Bits.u1TF)
4665	{
4666	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4667	* premature trips to ring-3 esp since IEM does not yet handle it. */
4668	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4669	AssertRCReturn(rc, rc);
4670	}
4671	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4672	* nested-guest VMCS. */
4673
4674	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4675	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4676	}
4677	return VINF_SUCCESS;
4678	}
4679
4680
4681	/**
4682	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4683	*
4684	* The guest FPU state is always pre-loaded hence we don't need to bother about
4685	* sharing FPU related CR0 bits between the guest and host.
4686	*
4687	* @returns VBox status code.
4688	* @param pVCpu The cross context virtual CPU structure.
4689	* @param pVmxTransient The VMX-transient structure.
4690	*
4691	* @remarks No-long-jump zone!!!
4692	*/
4693	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4694	{
4695	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4696	{
4697	PVM pVM = pVCpu->CTX_SUFF(pVM);
4698	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4699
4700	/*
4701	* Figure out fixed CR0 bits in VMX operation.
4702	*/
4703	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4704	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4705	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4706	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4707	else
4708	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4709
4710	if (!pVmxTransient->fIsNestedGuest)
4711	{
4712	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4713	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4714	uint64_t const u64ShadowCr0 = u64GuestCr0;
4715	Assert(!RT_HI_U32(u64GuestCr0));
4716
4717	/*
4718	* Setup VT-x's view of the guest CR0.
4719	*/
4720	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4721	if (pVM->hm.s.fNestedPaging)
4722	{
4723	if (CPUMIsGuestPagingEnabled(pVCpu))
4724	{
4725	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4726	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4727	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4728	}
4729	else
4730	{
4731	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4732	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4733	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4734	}
4735
4736	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4737	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4738	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4739	}
4740	else
4741	{
4742	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4743	u64GuestCr0 \|= X86_CR0_WP;
4744	}
4745
4746	/*
4747	* Guest FPU bits.
4748	*
4749	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4750	* using CR0.TS.
4751	*
4752	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4753	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4754	*/
4755	u64GuestCr0 \|= X86_CR0_NE;
4756
4757	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4758	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4759
4760	/*
4761	* Update exception intercepts.
4762	*/
4763	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4764	if (pVmcsInfo->RealMode.fRealOnV86Active)
4765	{
4766	Assert(PDMVmmDevHeapIsEnabled(pVM));
4767	Assert(pVM->hm.s.vmx.pRealModeTSS);
4768	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4769	}
4770	else
4771	{
4772	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4773	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4774	if (fInterceptMF)
4775	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4776	}
4777
4778	/* Additional intercepts for debugging, define these yourself explicitly. */
4779	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4780	uXcptBitmap \|= 0
4781	\| RT_BIT(X86_XCPT_BP)
4782	\| RT_BIT(X86_XCPT_DE)
4783	\| RT_BIT(X86_XCPT_NM)
4784	\| RT_BIT(X86_XCPT_TS)
4785	\| RT_BIT(X86_XCPT_UD)
4786	\| RT_BIT(X86_XCPT_NP)
4787	\| RT_BIT(X86_XCPT_SS)
4788	\| RT_BIT(X86_XCPT_GP)
4789	\| RT_BIT(X86_XCPT_PF)
4790	\| RT_BIT(X86_XCPT_MF)
4791	;
4792	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4793	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4794	#endif
4795	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4796	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4797	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4798
4799	/* Apply the fixed CR0 bits and enable caching. */
4800	u64GuestCr0 \|= fSetCr0;
4801	u64GuestCr0 &= fZapCr0;
4802	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4803
4804	/* Commit the CR0 and related fields to the guest VMCS. */
4805	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4806	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4807	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4808	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4809	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4810	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4811	AssertRCReturn(rc, rc);
4812
4813	/* Update our caches. */
4814	pVmcsInfo->u32ProcCtls = uProcCtls;
4815	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4816
4817	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4818	}
4819	else
4820	{
4821	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4822	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4823	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4824	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4825	Assert(!RT_HI_U32(u64GuestCr0));
4826	Assert(u64GuestCr0 & X86_CR0_NE);
4827
4828	/* Apply the fixed CR0 bits and enable caching. */
4829	u64GuestCr0 \|= fSetCr0;
4830	u64GuestCr0 &= fZapCr0;
4831	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4832
4833	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4834	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4835	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4836	AssertRCReturn(rc, rc);
4837
4838	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4839	}
4840
4841	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4842	}
4843
4844	return VINF_SUCCESS;
4845	}
4846
4847
4848	/**
4849	* Exports the guest control registers (CR3, CR4) into the guest-state area
4850	* in the VMCS.
4851	*
4852	* @returns VBox strict status code.
4853	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4854	* without unrestricted guest access and the VMMDev is not presently
4855	* mapped (e.g. EFI32).
4856	*
4857	* @param pVCpu The cross context virtual CPU structure.
4858	* @param pVmxTransient The VMX-transient structure.
4859	*
4860	* @remarks No-long-jump zone!!!
4861	*/
4862	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4863	{
4864	int rc = VINF_SUCCESS;
4865	PVM pVM = pVCpu->CTX_SUFF(pVM);
4866
4867	/*
4868	* Guest CR2.
4869	* It's always loaded in the assembler code. Nothing to do here.
4870	*/
4871
4872	/*
4873	* Guest CR3.
4874	*/
4875	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4876	{
4877	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4878
4879	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4880	if (pVM->hm.s.fNestedPaging)
4881	{
4882	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4883	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4884
4885	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4886	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4887	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4888	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4889
4890	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4891	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4892	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4893
4894	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4895	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4896	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4897	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4898	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4899	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4900	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4901
4902	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4903	AssertRCReturn(rc, rc);
4904
4905	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4906	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4907	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4908	{
4909	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4910	if (CPUMIsGuestInPAEModeEx(pCtx))
4911	{
4912	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4913	AssertRCReturn(rc, rc);
4914	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4915	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4916	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4917	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4918	AssertRCReturn(rc, rc);
4919	}
4920
4921	/*
4922	* The guest's view of its CR3 is unblemished with nested paging when the
4923	* guest is using paging or we have unrestricted guest execution to handle
4924	* the guest when it's not using paging.
4925	*/
4926	GCPhysGuestCR3 = pCtx->cr3;
4927	}
4928	else
4929	{
4930	/*
4931	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4932	* thinks it accesses physical memory directly, we use our identity-mapped
4933	* page table to map guest-linear to guest-physical addresses. EPT takes care
4934	* of translating it to host-physical addresses.
4935	*/
4936	RTGCPHYS GCPhys;
4937	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4938
4939	/* We obtain it here every time as the guest could have relocated this PCI region. */
4940	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4941	if (RT_SUCCESS(rc))
4942	{ /* likely */ }
4943	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4944	{
4945	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4946	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4947	}
4948	else
4949	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4950
4951	GCPhysGuestCR3 = GCPhys;
4952	}
4953
4954	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4955	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4956	AssertRCReturn(rc, rc);
4957	}
4958	else
4959	{
4960	/* Non-nested paging case, just use the hypervisor's CR3. */
4961	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4962
4963	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4964	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4965	AssertRCReturn(rc, rc);
4966	}
4967
4968	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4969	}
4970
4971	/*
4972	* Guest CR4.
4973	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4974	*/
4975	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4976	{
4977	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4978	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4979	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4980
4981	/*
4982	* Figure out fixed CR4 bits in VMX operation.
4983	*/
4984	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4985	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4986
4987	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4988	uint64_t u64GuestCr4 = pCtx->cr4;
4989	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4990	Assert(!RT_HI_U32(u64GuestCr4));
4991
4992	/*
4993	* Setup VT-x's view of the guest CR4.
4994	*
4995	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4996	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
4997	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4998	*
4999	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5000	*/
5001	if (pVmcsInfo->RealMode.fRealOnV86Active)
5002	{
5003	Assert(pVM->hm.s.vmx.pRealModeTSS);
5004	Assert(PDMVmmDevHeapIsEnabled(pVM));
5005	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5006	}
5007
5008	if (pVM->hm.s.fNestedPaging)
5009	{
5010	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5011	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5012	{
5013	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5014	u64GuestCr4 \|= X86_CR4_PSE;
5015	/* Our identity mapping is a 32-bit page directory. */
5016	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5017	}
5018	/* else use guest CR4.*/
5019	}
5020	else
5021	{
5022	Assert(!pVmxTransient->fIsNestedGuest);
5023
5024	/*
5025	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5026	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5027	*/
5028	switch (pVCpu->hm.s.enmShadowMode)
5029	{
5030	case PGMMODE_REAL: /* Real-mode. */
5031	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5032	case PGMMODE_32_BIT: /* 32-bit paging. */
5033	{
5034	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5035	break;
5036	}
5037
5038	case PGMMODE_PAE: /* PAE paging. */
5039	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5040	{
5041	u64GuestCr4 \|= X86_CR4_PAE;
5042	break;
5043	}
5044
5045	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5046	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5047	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5048	break;
5049	#endif
5050	default:
5051	AssertFailed();
5052	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5053	}
5054	}
5055
5056	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5057	u64GuestCr4 \|= fSetCr4;
5058	u64GuestCr4 &= fZapCr4;
5059
5060	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5061	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4); /** @todo Fix to 64-bit when we drop 32-bit. */
5062	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5063	AssertRCReturn(rc, rc);
5064
5065	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5066	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5067
5068	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5069
5070	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5071	}
5072	return rc;
5073	}
5074
5075
5076	/**
5077	* Exports the guest debug registers into the guest-state area in the VMCS.
5078	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5079	*
5080	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5081	*
5082	* @returns VBox status code.
5083	* @param pVCpu The cross context virtual CPU structure.
5084	* @param pVmxTransient The VMX-transient structure.
5085	*
5086	* @remarks No-long-jump zone!!!
5087	*/
5088	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5089	{
5090	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5091
5092	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5093	* stepping. */
5094	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5095	if (pVmxTransient->fIsNestedGuest)
5096	{
5097	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5098	AssertRCReturn(rc, rc);
5099	return VINF_SUCCESS;
5100	}
5101
5102	#ifdef VBOX_STRICT
5103	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5104	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5105	{
5106	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5107	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5108	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5109	}
5110	#endif
5111
5112	bool fSteppingDB = false;
5113	bool fInterceptMovDRx = false;
5114	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5115	if (pVCpu->hm.s.fSingleInstruction)
5116	{
5117	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5118	PVM pVM = pVCpu->CTX_SUFF(pVM);
5119	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5120	{
5121	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5122	Assert(fSteppingDB == false);
5123	}
5124	else
5125	{
5126	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5127	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5128	pVCpu->hm.s.fClearTrapFlag = true;
5129	fSteppingDB = true;
5130	}
5131	}
5132
5133	uint32_t u32GuestDr7;
5134	if ( fSteppingDB
5135	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5136	{
5137	/*
5138	* Use the combined guest and host DRx values found in the hypervisor register set
5139	* because the hypervisor debugger has breakpoints active or someone is single stepping
5140	* on the host side without a monitor trap flag.
5141	*
5142	* Note! DBGF expects a clean DR6 state before executing guest code.
5143	*/
5144	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5145	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5146	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5147	{
5148	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5149	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5150	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5151	}
5152	else
5153	#endif
5154	if (!CPUMIsHyperDebugStateActive(pVCpu))
5155	{
5156	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5157	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5158	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5159	}
5160
5161	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5162	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5163	pVCpu->hm.s.fUsingHyperDR7 = true;
5164	fInterceptMovDRx = true;
5165	}
5166	else
5167	{
5168	/*
5169	* If the guest has enabled debug registers, we need to load them prior to
5170	* executing guest code so they'll trigger at the right time.
5171	*/
5172	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5173	{
5174	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5175	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5176	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5177	{
5178	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5179	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5180	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5181	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5182	}
5183	else
5184	#endif
5185	if (!CPUMIsGuestDebugStateActive(pVCpu))
5186	{
5187	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5188	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5189	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5190	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5191	}
5192	Assert(!fInterceptMovDRx);
5193	}
5194	/*
5195	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5196	* must intercept #DB in order to maintain a correct DR6 guest value, and
5197	* because we need to intercept it to prevent nested #DBs from hanging the
5198	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5199	*/
5200	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5201	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5202	&& !CPUMIsGuestDebugStateActive(pVCpu))
5203	#else
5204	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5205	#endif
5206	{
5207	fInterceptMovDRx = true;
5208	}
5209
5210	/* Update DR7 with the actual guest value. */
5211	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5212	pVCpu->hm.s.fUsingHyperDR7 = false;
5213	}
5214
5215	if (fInterceptMovDRx)
5216	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5217	else
5218	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5219
5220	/*
5221	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5222	* monitor-trap flag and update our cache.
5223	*/
5224	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5225	{
5226	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5227	AssertRCReturn(rc2, rc2);
5228	pVmcsInfo->u32ProcCtls = uProcCtls;
5229	}
5230
5231	/*
5232	* Update guest DR7.
5233	*/
5234	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5235	AssertRCReturn(rc, rc);
5236
5237	/*
5238	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5239	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5240	*
5241	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5242	*/
5243	if (fSteppingDB)
5244	{
5245	Assert(pVCpu->hm.s.fSingleInstruction);
5246	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5247
5248	uint32_t fIntrState = 0;
5249	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5250	AssertRCReturn(rc, rc);
5251
5252	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5253	{
5254	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5255	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5256	AssertRCReturn(rc, rc);
5257	}
5258	}
5259
5260	return VINF_SUCCESS;
5261	}
5262
5263
5264	#ifdef VBOX_STRICT
5265	/**
5266	* Strict function to validate segment registers.
5267	*
5268	* @param pVCpu The cross context virtual CPU structure.
5269	* @param pVmcsInfo The VMCS info. object.
5270	*
5271	* @remarks Will import guest CR0 on strict builds during validation of
5272	* segments.
5273	*/
5274	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5275	{
5276	/*
5277	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5278	*
5279	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5280	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5281	* unusable bit and doesn't change the guest-context value.
5282	*/
5283	PVM pVM = pVCpu->CTX_SUFF(pVM);
5284	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5285	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5286	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5287	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5288	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5289	{
5290	/* Protected mode checks */
5291	/* CS */
5292	Assert(pCtx->cs.Attr.n.u1Present);
5293	Assert(!(pCtx->cs.Attr.u & 0xf00));
5294	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5295	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5296	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5297	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5298	\|\| (pCtx->cs.Attr.n.u1Granularity));
5299	/* CS cannot be loaded with NULL in protected mode. */
5300	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5301	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5302	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5303	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5304	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5305	else
5306	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5307	/* SS */
5308	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5309	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5310	if ( !(pCtx->cr0 & X86_CR0_PE)
5311	\|\| pCtx->cs.Attr.n.u4Type == 3)
5312	{
5313	Assert(!pCtx->ss.Attr.n.u2Dpl);
5314	}
5315	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5316	{
5317	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5318	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5319	Assert(pCtx->ss.Attr.n.u1Present);
5320	Assert(!(pCtx->ss.Attr.u & 0xf00));
5321	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5322	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5323	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5324	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5325	\|\| (pCtx->ss.Attr.n.u1Granularity));
5326	}
5327	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5328	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5329	{
5330	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5331	Assert(pCtx->ds.Attr.n.u1Present);
5332	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5333	Assert(!(pCtx->ds.Attr.u & 0xf00));
5334	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5335	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5336	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5337	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5338	\|\| (pCtx->ds.Attr.n.u1Granularity));
5339	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5340	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5341	}
5342	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5343	{
5344	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5345	Assert(pCtx->es.Attr.n.u1Present);
5346	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5347	Assert(!(pCtx->es.Attr.u & 0xf00));
5348	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5349	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5350	\|\| !(pCtx->es.Attr.n.u1Granularity));
5351	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5352	\|\| (pCtx->es.Attr.n.u1Granularity));
5353	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5354	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5355	}
5356	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5357	{
5358	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5359	Assert(pCtx->fs.Attr.n.u1Present);
5360	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5361	Assert(!(pCtx->fs.Attr.u & 0xf00));
5362	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5363	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5364	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5365	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5366	\|\| (pCtx->fs.Attr.n.u1Granularity));
5367	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5368	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5369	}
5370	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5371	{
5372	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5373	Assert(pCtx->gs.Attr.n.u1Present);
5374	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5375	Assert(!(pCtx->gs.Attr.u & 0xf00));
5376	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5377	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5378	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5379	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5380	\|\| (pCtx->gs.Attr.n.u1Granularity));
5381	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5382	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5383	}
5384	/* 64-bit capable CPUs. */
5385	# if HC_ARCH_BITS == 64
5386	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5387	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5388	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5389	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5390	# endif
5391	}
5392	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5393	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5394	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5395	{
5396	/* Real and v86 mode checks. */
5397	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5398	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5399	if (pVmcsInfo->RealMode.fRealOnV86Active)
5400	{
5401	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5402	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5403	}
5404	else
5405	{
5406	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5407	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5408	}
5409
5410	/* CS */
5411	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5412	Assert(pCtx->cs.u32Limit == 0xffff);
5413	Assert(u32CSAttr == 0xf3);
5414	/* SS */
5415	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5416	Assert(pCtx->ss.u32Limit == 0xffff);
5417	Assert(u32SSAttr == 0xf3);
5418	/* DS */
5419	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5420	Assert(pCtx->ds.u32Limit == 0xffff);
5421	Assert(u32DSAttr == 0xf3);
5422	/* ES */
5423	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5424	Assert(pCtx->es.u32Limit == 0xffff);
5425	Assert(u32ESAttr == 0xf3);
5426	/* FS */
5427	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5428	Assert(pCtx->fs.u32Limit == 0xffff);
5429	Assert(u32FSAttr == 0xf3);
5430	/* GS */
5431	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5432	Assert(pCtx->gs.u32Limit == 0xffff);
5433	Assert(u32GSAttr == 0xf3);
5434	/* 64-bit capable CPUs. */
5435	# if HC_ARCH_BITS == 64
5436	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5437	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5438	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5439	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5440	# endif
5441	}
5442	}
5443	#endif /* VBOX_STRICT */
5444
5445
5446	/**
5447	* Exports a guest segment register into the guest-state area in the VMCS.
5448	*
5449	* @returns VBox status code.
5450	* @param pVCpu The cross context virtual CPU structure.
5451	* @param pVmcsInfo The VMCS info. object.
5452	* @param iSegReg The segment register number (X86_SREG_XXX).
5453	* @param pSelReg Pointer to the segment selector.
5454	*
5455	* @remarks No-long-jump zone!!!
5456	*/
5457	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5458	{
5459	Assert(iSegReg < X86_SREG_COUNT);
5460	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5461	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5462	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5463	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5464
5465	uint32_t u32Access = pSelReg->Attr.u;
5466	if (pVmcsInfo->RealMode.fRealOnV86Active)
5467	{
5468	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5469	u32Access = 0xf3;
5470	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5471	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5472	RT_NOREF_PV(pVCpu);
5473	}
5474	else
5475	{
5476	/*
5477	* The way to differentiate between whether this is really a null selector or was just
5478	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5479	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5480	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5481	* NULL selectors loaded in protected-mode have their attribute as 0.
5482	*/
5483	if (!u32Access)
5484	u32Access = X86DESCATTR_UNUSABLE;
5485	}
5486
5487	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5488	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5489	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5490
5491	/*
5492	* Commit it to the VMCS.
5493	*/
5494	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5495	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5496	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5497	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5498	AssertRCReturn(rc, rc);
5499	return rc;
5500	}
5501
5502
5503	/**
5504	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5505	* area in the VMCS.
5506	*
5507	* @returns VBox status code.
5508	* @param pVCpu The cross context virtual CPU structure.
5509	* @param pVmxTransient The VMX-transient structure.
5510	*
5511	* @remarks Will import guest CR0 on strict builds during validation of
5512	* segments.
5513	* @remarks No-long-jump zone!!!
5514	*/
5515	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5516	{
5517	int rc = VERR_INTERNAL_ERROR_5;
5518	PVM pVM = pVCpu->CTX_SUFF(pVM);
5519	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5520	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5521
5522	/*
5523	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5524	*/
5525	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5526	{
5527	#ifdef VBOX_WITH_REM
5528	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5529	{
5530	Assert(!pVmxTransient->fIsNestedGuest);
5531	Assert(pVM->hm.s.vmx.pRealModeTSS);
5532	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5533	if ( pVmcsInfo->fWasInRealMode
5534	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5535	{
5536	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5537	in real-mode (e.g. OpenBSD 4.0) */
5538	REMFlushTBs(pVM);
5539	Log4Func(("Switch to protected mode detected!\n"));
5540	pVmcsInfo->fWasInRealMode = false;
5541	}
5542	}
5543	#endif
5544	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5545	{
5546	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5547	if (pVmcsInfo->RealMode.fRealOnV86Active)
5548	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5549	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5550	AssertRCReturn(rc, rc);
5551	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5552	}
5553
5554	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5555	{
5556	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5557	if (pVmcsInfo->RealMode.fRealOnV86Active)
5558	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5559	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5560	AssertRCReturn(rc, rc);
5561	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5562	}
5563
5564	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5565	{
5566	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5567	if (pVmcsInfo->RealMode.fRealOnV86Active)
5568	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5569	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5570	AssertRCReturn(rc, rc);
5571	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5572	}
5573
5574	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5575	{
5576	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5577	if (pVmcsInfo->RealMode.fRealOnV86Active)
5578	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5579	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5580	AssertRCReturn(rc, rc);
5581	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5582	}
5583
5584	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5585	{
5586	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5587	if (pVmcsInfo->RealMode.fRealOnV86Active)
5588	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5589	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5590	AssertRCReturn(rc, rc);
5591	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5592	}
5593
5594	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5595	{
5596	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5597	if (pVmcsInfo->RealMode.fRealOnV86Active)
5598	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5599	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5600	AssertRCReturn(rc, rc);
5601	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5602	}
5603
5604	#ifdef VBOX_STRICT
5605	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5606	#endif
5607	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5608	pCtx->cs.Attr.u));
5609	}
5610
5611	/*
5612	* Guest TR.
5613	*/
5614	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5615	{
5616	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5617
5618	/*
5619	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5620	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5621	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5622	*/
5623	uint16_t u16Sel;
5624	uint32_t u32Limit;
5625	uint64_t u64Base;
5626	uint32_t u32AccessRights;
5627	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5628	{
5629	u16Sel = pCtx->tr.Sel;
5630	u32Limit = pCtx->tr.u32Limit;
5631	u64Base = pCtx->tr.u64Base;
5632	u32AccessRights = pCtx->tr.Attr.u;
5633	}
5634	else
5635	{
5636	Assert(!pVmxTransient->fIsNestedGuest);
5637	Assert(pVM->hm.s.vmx.pRealModeTSS);
5638	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5639
5640	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5641	RTGCPHYS GCPhys;
5642	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5643	AssertRCReturn(rc, rc);
5644
5645	X86DESCATTR DescAttr;
5646	DescAttr.u = 0;
5647	DescAttr.n.u1Present = 1;
5648	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5649
5650	u16Sel = 0;
5651	u32Limit = HM_VTX_TSS_SIZE;
5652	u64Base = GCPhys;
5653	u32AccessRights = DescAttr.u;
5654	}
5655
5656	/* Validate. */
5657	Assert(!(u16Sel & RT_BIT(2)));
5658	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5659	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5660	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5661	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5662	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5663	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5664	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5665	Assert( (u32Limit & 0xfff) == 0xfff
5666	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5667	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5668	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5669
5670	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5671	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5672	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5673	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5674	AssertRCReturn(rc, rc);
5675
5676	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5677	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5678	}
5679
5680	/*
5681	* Guest GDTR.
5682	*/
5683	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5684	{
5685	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5686
5687	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5688	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5689	AssertRCReturn(rc, rc);
5690
5691	/* Validate. */
5692	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5693
5694	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5695	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5696	}
5697
5698	/*
5699	* Guest LDTR.
5700	*/
5701	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5702	{
5703	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5704
5705	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5706	uint32_t u32Access;
5707	if ( !pVmxTransient->fIsNestedGuest
5708	&& !pCtx->ldtr.Attr.u)
5709	u32Access = X86DESCATTR_UNUSABLE;
5710	else
5711	u32Access = pCtx->ldtr.Attr.u;
5712
5713	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5714	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5715	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5716	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5717	AssertRCReturn(rc, rc);
5718
5719	/* Validate. */
5720	if (!(u32Access & X86DESCATTR_UNUSABLE))
5721	{
5722	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5723	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5724	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5725	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5726	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5727	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5728	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5729	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5730	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5731	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5732	}
5733
5734	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5735	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5736	}
5737
5738	/*
5739	* Guest IDTR.
5740	*/
5741	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5742	{
5743	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5744
5745	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5746	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5747	AssertRCReturn(rc, rc);
5748
5749	/* Validate. */
5750	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5751
5752	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5753	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5754	}
5755
5756	return VINF_SUCCESS;
5757	}
5758
5759
5760	/**
5761	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5762	* areas.
5763	*
5764	* These MSRs will automatically be loaded to the host CPU on every successful
5765	* VM-entry and stored from the host CPU on every successful VM-exit.
5766	*
5767	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5768	* actual host MSR values are not- updated here for performance reasons. See
5769	* hmR0VmxExportHostMsrs().
5770	*
5771	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5772	*
5773	* @returns VBox status code.
5774	* @param pVCpu The cross context virtual CPU structure.
5775	* @param pVmxTransient The VMX-transient structure.
5776	*
5777	* @remarks No-long-jump zone!!!
5778	*/
5779	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5780	{
5781	AssertPtr(pVCpu);
5782	AssertPtr(pVmxTransient);
5783
5784	PVM pVM = pVCpu->CTX_SUFF(pVM);
5785	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5786
5787	/*
5788	* MSRs that we use the auto-load/store MSR area in the VMCS.
5789	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5790	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5791	*
5792	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5793	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5794	* emulation, nothing to do here.
5795	*/
5796	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5797	{
5798	if ( !pVmxTransient->fIsNestedGuest
5799	&& pVM->hm.s.fAllow64BitGuests)
5800	{
5801	#if HC_ARCH_BITS == 32
5802	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5803	Assert(!pVmxTransient->fIsNestedGuest);
5804
5805	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5806	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5807	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5808	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5809	AssertRCReturn(rc, rc);
5810	#endif
5811	}
5812	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5813	}
5814
5815	/*
5816	* Guest Sysenter MSRs.
5817	*/
5818	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5819	{
5820	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5821
5822	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5823	{
5824	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5825	AssertRCReturn(rc, rc);
5826	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5827	}
5828
5829	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5830	{
5831	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5832	AssertRCReturn(rc, rc);
5833	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5834	}
5835
5836	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5837	{
5838	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5839	AssertRCReturn(rc, rc);
5840	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5841	}
5842	}
5843
5844	/*
5845	* Guest/host EFER MSR.
5846	*/
5847	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5848	{
5849	/* Whether we are using the VMCS to swap the EFER MSR must have been
5850	determined earlier while exporting VM-entry/VM-exit controls. */
5851	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5852	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5853
5854	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5855	{
5856	/*
5857	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5858	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5859	*/
5860	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5861	{
5862	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5863	AssertRCReturn(rc, rc);
5864	}
5865	else
5866	{
5867	/*
5868	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5869	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5870	*/
5871	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5872	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5873	AssertRCReturn(rc, rc);
5874	}
5875	}
5876	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5877	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5878
5879	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5880	}
5881
5882	/*
5883	* Other MSRs.
5884	* Speculation Control (R/W).
5885	*/
5886	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5887	{
5888	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5889	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5890	{
5891	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5892	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5893	AssertRCReturn(rc, rc);
5894	}
5895	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5896	}
5897
5898	return VINF_SUCCESS;
5899	}
5900
5901
5902	/**
5903	* Selects up the appropriate function to run guest code.
5904	*
5905	* @returns VBox status code.
5906	* @param pVCpu The cross context virtual CPU structure.
5907	* @param pVmxTransient The VMX-transient structure.
5908	*
5909	* @remarks No-long-jump zone!!!
5910	*/
5911	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5912	{
5913	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5914	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5915
5916	if (CPUMIsGuestInLongModeEx(pCtx))
5917	{
5918	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5919	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5920	#endif
5921	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5922	#if HC_ARCH_BITS == 32
5923	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5924	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5925	{
5926	#ifdef VBOX_STRICT
5927	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5928	{
5929	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5930	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5931	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5932	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5933	("fCtxChanged=%#RX64\n", fCtxChanged));
5934	}
5935	#endif
5936	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5937
5938	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5939	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5940	pVmcsInfo->fSwitchedTo64on32 = true;
5941	Log4Func(("Selected 64-bit switcher\n"));
5942	}
5943	#else
5944	/* 64-bit host. */
5945	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5946	#endif
5947	}
5948	else
5949	{
5950	/* Guest is not in long mode, use the 32-bit handler. */
5951	#if HC_ARCH_BITS == 32
5952	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5953	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5954	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5955	{
5956	# ifdef VBOX_STRICT
5957	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5958	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5959	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5960	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5961	("fCtxChanged=%#RX64\n", fCtxChanged));
5962	# endif
5963	}
5964	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5965	/*
5966	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5967	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5968	* switcher flag now because we know the guest is in a sane state where it's safe
5969	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5970	* the much faster 32-bit switcher again.
5971	*/
5972	if (!pVmcsInfo->fSwitchedTo64on32)
5973	{
5974	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5975	Log4Func(("Selected 32-bit switcher\n"));
5976	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5977	}
5978	else
5979	{
5980	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5981	if ( pVmcsInfo->RealMode.fRealOnV86Active
5982	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5983	{
5984	pVmcsInfo->fSwitchedTo64on32 = false;
5985	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5986	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5987	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5988	\| HM_CHANGED_HOST_CONTEXT);
5989	Log4Func(("Selected 32-bit switcher (safe)\n"));
5990	}
5991	}
5992	# else
5993	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5994	# endif
5995	#else
5996	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5997	#endif
5998	}
5999	Assert(pVmcsInfo->pfnStartVM);
6000	return VINF_SUCCESS;
6001	}
6002
6003
6004	/**
6005	* Wrapper for running the guest code in VT-x.
6006	*
6007	* @returns VBox status code, no informational status codes.
6008	* @param pVCpu The cross context virtual CPU structure.
6009	* @param pVmxTransient The VMX-transient structure.
6010	*
6011	* @remarks No-long-jump zone!!!
6012	*/
6013	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6014	{
6015	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6016	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6017	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6018
6019	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6020
6021	/*
6022	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6023	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6024	* callee-saved and thus the need for this XMM wrapper.
6025	*
6026	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6027	*/
6028	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6029	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6030	PVM pVM = pVCpu->CTX_SUFF(pVM);
6031	#ifdef VBOX_WITH_KERNEL_USING_XMM
6032	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6033	#else
6034	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6035	#endif
6036	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6037	return rc;
6038	}
6039
6040
6041	/**
6042	* Reports world-switch error and dumps some useful debug info.
6043	*
6044	* @param pVCpu The cross context virtual CPU structure.
6045	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6046	* @param pVmxTransient The VMX-transient structure (only
6047	* exitReason updated).
6048	*/
6049	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6050	{
6051	Assert(pVCpu);
6052	Assert(pVmxTransient);
6053	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6054
6055	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6056	switch (rcVMRun)
6057	{
6058	case VERR_VMX_INVALID_VMXON_PTR:
6059	AssertFailed();
6060	break;
6061	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6062	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6063	{
6064	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6065	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6066	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6067	AssertRC(rc);
6068
6069	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6070	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6071	Cannot do it here as we may have been long preempted. */
6072
6073	#ifdef VBOX_STRICT
6074	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6075	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6076	pVmxTransient->uExitReason));
6077	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6078	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6079	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6080	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6081	else
6082	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6083	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6084	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6085
6086	/* VMX control bits. */
6087	uint32_t u32Val;
6088	uint64_t u64Val;
6089	RTHCUINTREG uHCReg;
6090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6091	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6093	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6094	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6095	{
6096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6097	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6098	}
6099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6100	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6102	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6104	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6106	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6108	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6110	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6112	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6114	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6116	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6118	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6120	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6122	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6124	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6125	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6126	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6127	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6128	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6129	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6130	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6131	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6132	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6133	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6134	{
6135	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6136	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6137	}
6138
6139	/* Guest bits. */
6140	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6141	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6142	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6143	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6144	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6145	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6146	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6147	{
6148	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6149	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6150	}
6151
6152	/* Host bits. */
6153	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6154	Log4(("Host CR0 %#RHr\n", uHCReg));
6155	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6156	Log4(("Host CR3 %#RHr\n", uHCReg));
6157	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6158	Log4(("Host CR4 %#RHr\n", uHCReg));
6159
6160	RTGDTR HostGdtr;
6161	PCX86DESCHC pDesc;
6162	ASMGetGDTR(&HostGdtr);
6163	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6164	Log4(("Host CS %#08x\n", u32Val));
6165	if (u32Val < HostGdtr.cbGdt)
6166	{
6167	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6168	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6169	}
6170
6171	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6172	Log4(("Host DS %#08x\n", u32Val));
6173	if (u32Val < HostGdtr.cbGdt)
6174	{
6175	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6176	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6177	}
6178
6179	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6180	Log4(("Host ES %#08x\n", u32Val));
6181	if (u32Val < HostGdtr.cbGdt)
6182	{
6183	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6184	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6185	}
6186
6187	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6188	Log4(("Host FS %#08x\n", u32Val));
6189	if (u32Val < HostGdtr.cbGdt)
6190	{
6191	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6192	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6193	}
6194
6195	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6196	Log4(("Host GS %#08x\n", u32Val));
6197	if (u32Val < HostGdtr.cbGdt)
6198	{
6199	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6200	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6201	}
6202
6203	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6204	Log4(("Host SS %#08x\n", u32Val));
6205	if (u32Val < HostGdtr.cbGdt)
6206	{
6207	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6208	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6209	}
6210
6211	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6212	Log4(("Host TR %#08x\n", u32Val));
6213	if (u32Val < HostGdtr.cbGdt)
6214	{
6215	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6216	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6217	}
6218
6219	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6220	Log4(("Host TR Base %#RHv\n", uHCReg));
6221	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6222	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6224	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6225	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6226	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6228	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6229	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6230	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6231	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6232	Log4(("Host RSP %#RHv\n", uHCReg));
6233	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6234	Log4(("Host RIP %#RHv\n", uHCReg));
6235	# if HC_ARCH_BITS == 64
6236	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6237	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6238	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6239	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6240	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6241	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6242	# endif
6243	#endif /* VBOX_STRICT */
6244	break;
6245	}
6246
6247	default:
6248	/* Impossible */
6249	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6250	break;
6251	}
6252	}
6253
6254
6255	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6256	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6257	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6258	# endif
6259
6260	/**
6261	* Initialize the VMCS-Read cache.
6262	*
6263	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6264	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6265	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6266	* (those that have a 32-bit FULL & HIGH part).
6267	*
6268	* @param pVCpu The cross context virtual CPU structure.
6269	*/
6270	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6271	{
6272	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6273	do { \
6274	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6275	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6276	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6277	++cReadFields; \
6278	} while (0)
6279
6280	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6281	uint32_t cReadFields = 0;
6282
6283	/*
6284	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6285	* and serve to indicate exceptions to the rules.
6286	*/
6287
6288	/* Guest-natural selector base fields. */
6289	#if 0
6290	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6293	#endif
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6306	#if 0
6307	/* Unused natural width guest-state fields. */
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6310	#endif
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6313
6314	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6315	these 64-bit fields (using "FULL" and "HIGH" fields). */
6316	#if 0
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6326	#endif
6327
6328	/* Natural width guest-state fields. */
6329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6331
6332	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6333	{
6334	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6335	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6336	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6337	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6338	}
6339	else
6340	{
6341	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6342	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6343	}
6344
6345	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6346	}
6347
6348
6349	/**
6350	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6351	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6352	* darwin, running 64-bit guests).
6353	*
6354	* @returns VBox status code.
6355	* @param pVCpu The cross context virtual CPU structure.
6356	* @param idxField The VMCS field encoding.
6357	* @param u64Val 16, 32 or 64-bit value.
6358	*/
6359	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6360	{
6361	int rc;
6362	switch (idxField)
6363	{
6364	/*
6365	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6366	*/
6367	/* 64-bit Control fields. */
6368	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6369	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6370	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6371	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6372	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6373	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6374	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6375	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6376	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6377	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6378	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6379	case VMX_VMCS64_CTRL_EPTP_FULL:
6380	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6381	/* 64-bit Guest-state fields. */
6382	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6383	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6384	case VMX_VMCS64_GUEST_PAT_FULL:
6385	case VMX_VMCS64_GUEST_EFER_FULL:
6386	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6387	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6388	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6389	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6390	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6391	/* 64-bit Host-state fields. */
6392	case VMX_VMCS64_HOST_PAT_FULL:
6393	case VMX_VMCS64_HOST_EFER_FULL:
6394	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6395	{
6396	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6397	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6398	break;
6399	}
6400
6401	/*
6402	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6403	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6404	*/
6405	/* Natural-width Guest-state fields. */
6406	case VMX_VMCS_GUEST_CR3:
6407	case VMX_VMCS_GUEST_ES_BASE:
6408	case VMX_VMCS_GUEST_CS_BASE:
6409	case VMX_VMCS_GUEST_SS_BASE:
6410	case VMX_VMCS_GUEST_DS_BASE:
6411	case VMX_VMCS_GUEST_FS_BASE:
6412	case VMX_VMCS_GUEST_GS_BASE:
6413	case VMX_VMCS_GUEST_LDTR_BASE:
6414	case VMX_VMCS_GUEST_TR_BASE:
6415	case VMX_VMCS_GUEST_GDTR_BASE:
6416	case VMX_VMCS_GUEST_IDTR_BASE:
6417	case VMX_VMCS_GUEST_RSP:
6418	case VMX_VMCS_GUEST_RIP:
6419	case VMX_VMCS_GUEST_SYSENTER_ESP:
6420	case VMX_VMCS_GUEST_SYSENTER_EIP:
6421	{
6422	if (!(RT_HI_U32(u64Val)))
6423	{
6424	/* If this field is 64-bit, VT-x will zero out the top bits. */
6425	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6426	}
6427	else
6428	{
6429	/* Assert that only the 32->64 switcher case should ever come here. */
6430	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6431	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6432	}
6433	break;
6434	}
6435
6436	default:
6437	{
6438	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6439	pVCpu->hm.s.u32HMError = idxField;
6440	rc = VERR_INVALID_PARAMETER;
6441	break;
6442	}
6443	}
6444	AssertRCReturn(rc, rc);
6445	return rc;
6446	}
6447
6448
6449	/**
6450	* Queue up a VMWRITE by using the VMCS write cache.
6451	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6452	*
6453	* @param pVCpu The cross context virtual CPU structure.
6454	* @param idxField The VMCS field encoding.
6455	* @param u64Val 16, 32 or 64-bit value.
6456	*/
6457	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6458	{
6459	AssertPtr(pVCpu);
6460	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6461
6462	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6463	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6464
6465	/* Make sure there are no duplicates. */
6466	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6467	{
6468	if (pCache->Write.aField[i] == idxField)
6469	{
6470	pCache->Write.aFieldVal[i] = u64Val;
6471	return VINF_SUCCESS;
6472	}
6473	}
6474
6475	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6476	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6477	pCache->Write.cValidEntries++;
6478	return VINF_SUCCESS;
6479	}
6480	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6481
6482
6483	/**
6484	* Sets up the usage of TSC-offsetting and updates the VMCS.
6485	*
6486	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6487	* VMX-preemption timer.
6488	*
6489	* @returns VBox status code.
6490	* @param pVCpu The cross context virtual CPU structure.
6491	* @param pVmxTransient The VMX-transient structure.
6492	*
6493	* @remarks No-long-jump zone!!!
6494	*/
6495	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6496	{
6497	bool fOffsettedTsc;
6498	bool fParavirtTsc;
6499	uint64_t uTscOffset;
6500	PVM pVM = pVCpu->CTX_SUFF(pVM);
6501	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6502
6503	if (pVM->hm.s.vmx.fUsePreemptTimer)
6504	{
6505	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6506
6507	/* Make sure the returned values have sane upper and lower boundaries. */
6508	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6509	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6510	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6511	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6512
6513	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6514	* preemption timers here. We probably need to clamp the preemption timer,
6515	* after converting the timer value to the host. */
6516	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6517	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6518	AssertRC(rc);
6519	}
6520	else
6521	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6522
6523	if (fParavirtTsc)
6524	{
6525	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6526	information before every VM-entry, hence disable it for performance sake. */
6527	#if 0
6528	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6529	AssertRC(rc);
6530	#endif
6531	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6532	}
6533
6534	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6535	if ( fOffsettedTsc
6536	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6537	{
6538	if (pVmxTransient->fIsNestedGuest)
6539	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6540	if (pVmcsInfo->u64TscOffset != uTscOffset)
6541	{
6542	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6543	AssertRC(rc);
6544	pVmcsInfo->u64TscOffset = uTscOffset;
6545	}
6546
6547	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6548	{
6549	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6550	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6551	AssertRC(rc);
6552	pVmcsInfo->u32ProcCtls = uProcCtls;
6553	}
6554	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6555	}
6556	else
6557	{
6558	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6559	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6560	{
6561	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6562	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6563	AssertRC(rc);
6564	pVmcsInfo->u32ProcCtls = uProcCtls;
6565	}
6566	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6567	}
6568	}
6569
6570
6571	/**
6572	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6573	* VM-exit interruption info type.
6574	*
6575	* @returns The IEM exception flags.
6576	* @param uVector The event vector.
6577	* @param uVmxEventType The VMX event type.
6578	*
6579	* @remarks This function currently only constructs flags required for
6580	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6581	* and CR2 aspects of an exception are not included).
6582	*/
6583	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6584	{
6585	uint32_t fIemXcptFlags;
6586	switch (uVmxEventType)
6587	{
6588	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6589	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6590	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6591	break;
6592
6593	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6594	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6595	break;
6596
6597	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6598	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6599	break;
6600
6601	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6602	{
6603	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6604	if (uVector == X86_XCPT_BP)
6605	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6606	else if (uVector == X86_XCPT_OF)
6607	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6608	else
6609	{
6610	fIemXcptFlags = 0;
6611	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6612	}
6613	break;
6614	}
6615
6616	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6617	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6618	break;
6619
6620	default:
6621	fIemXcptFlags = 0;
6622	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6623	break;
6624	}
6625	return fIemXcptFlags;
6626	}
6627
6628
6629	/**
6630	* Sets an event as a pending event to be injected into the guest.
6631	*
6632	* @param pVCpu The cross context virtual CPU structure.
6633	* @param u32IntInfo The VM-entry interruption-information field.
6634	* @param cbInstr The VM-entry instruction length in bytes (for software
6635	* interrupts, exceptions and privileged software
6636	* exceptions).
6637	* @param u32ErrCode The VM-entry exception error code.
6638	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6639	* page-fault.
6640	*/
6641	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6642	RTGCUINTPTR GCPtrFaultAddress)
6643	{
6644	Assert(!pVCpu->hm.s.Event.fPending);
6645	pVCpu->hm.s.Event.fPending = true;
6646	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6647	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6648	pVCpu->hm.s.Event.cbInstr = cbInstr;
6649	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6650	}
6651
6652
6653	/**
6654	* Sets an external interrupt as pending-for-injection into the VM.
6655	*
6656	* @param pVCpu The cross context virtual CPU structure.
6657	* @param u8Interrupt The external interrupt vector.
6658	*/
6659	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6660	{
6661	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6662	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6663	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6664	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6665	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6666	}
6667
6668
6669	/**
6670	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6671	*
6672	* @param pVCpu The cross context virtual CPU structure.
6673	*/
6674	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6675	{
6676	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6677	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6678	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6679	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6680	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6681	}
6682
6683
6684	/**
6685	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6686	*
6687	* @param pVCpu The cross context virtual CPU structure.
6688	*/
6689	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6690	{
6691	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6692	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6693	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6694	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6695	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6696	}
6697
6698
6699	/**
6700	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6701	*
6702	* @param pVCpu The cross context virtual CPU structure.
6703	*/
6704	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6705	{
6706	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6707	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6708	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6709	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6710	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6711	}
6712
6713
6714	/**
6715	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6716	*
6717	* @param pVCpu The cross context virtual CPU structure.
6718	*/
6719	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6720	{
6721	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6722	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6723	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6724	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6725	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6726	}
6727
6728
6729	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6730	/**
6731	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6732	*
6733	* @param pVCpu The cross context virtual CPU structure.
6734	* @param u32ErrCode The error code for the general-protection exception.
6735	*/
6736	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6737	{
6738	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6739	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6740	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6741	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6742	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6743	}
6744
6745
6746	/**
6747	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6748	*
6749	* @param pVCpu The cross context virtual CPU structure.
6750	* @param u32ErrCode The error code for the stack exception.
6751	*/
6752	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6753	{
6754	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6755	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6756	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6757	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6758	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6759	}
6760
6761
6762	/**
6763	* Decodes the memory operand of an instruction that caused a VM-exit.
6764	*
6765	* The VM-exit qualification field provides the displacement field for memory
6766	* operand instructions, if any.
6767	*
6768	* @returns Strict VBox status code (i.e. informational status codes too).
6769	* @retval VINF_SUCCESS if the operand was successfully decoded.
6770	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6771	* operand.
6772	* @param pVCpu The cross context virtual CPU structure.
6773	* @param uExitInstrInfo The VM-exit instruction information field.
6774	* @param enmMemAccess The memory operand's access type (read or write).
6775	* @param GCPtrDisp The instruction displacement field, if any. For
6776	* RIP-relative addressing pass RIP + displacement here.
6777	* @param pGCPtrMem Where to store the effective destination memory address.
6778	*/
6779	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6780	PRTGCPTR pGCPtrMem)
6781	{
6782	Assert(pGCPtrMem);
6783	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6784	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6785	\| CPUMCTX_EXTRN_CR0);
6786
6787	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6788	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6789	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6790
6791	VMXEXITINSTRINFO ExitInstrInfo;
6792	ExitInstrInfo.u = uExitInstrInfo;
6793	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6794	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6795	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6796	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6797	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6798	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6799	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6800	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6801	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6802
6803	/*
6804	* Validate instruction information.
6805	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6806	*/
6807	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6808	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6809	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6810	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6811	AssertLogRelMsgReturn(fIsMemOperand,
6812	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6813
6814	/*
6815	* Compute the complete effective address.
6816	*
6817	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6818	* See AMD spec. 4.5.2 "Segment Registers".
6819	*/
6820	RTGCPTR GCPtrMem = GCPtrDisp;
6821	if (fBaseRegValid)
6822	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6823	if (fIdxRegValid)
6824	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6825
6826	RTGCPTR const GCPtrOff = GCPtrMem;
6827	if ( !fIsLongMode
6828	\|\| iSegReg >= X86_SREG_FS)
6829	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6830	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6831
6832	/*
6833	* Validate effective address.
6834	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6835	*/
6836	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6837	Assert(cbAccess > 0);
6838	if (fIsLongMode)
6839	{
6840	if (X86_IS_CANONICAL(GCPtrMem))
6841	{
6842	*pGCPtrMem = GCPtrMem;
6843	return VINF_SUCCESS;
6844	}
6845
6846	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6847	* "Data Limit Checks in 64-bit Mode". */
6848	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6849	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6850	return VINF_HM_PENDING_XCPT;
6851	}
6852
6853	/*
6854	* This is a watered down version of iemMemApplySegment().
6855	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6856	* and segment CPL/DPL checks are skipped.
6857	*/
6858	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6859	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6860	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6861
6862	/* Check if the segment is present and usable. */
6863	if ( pSel->Attr.n.u1Present
6864	&& !pSel->Attr.n.u1Unusable)
6865	{
6866	Assert(pSel->Attr.n.u1DescType);
6867	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6868	{
6869	/* Check permissions for the data segment. */
6870	if ( enmMemAccess == VMXMEMACCESS_WRITE
6871	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6872	{
6873	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6874	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6875	return VINF_HM_PENDING_XCPT;
6876	}
6877
6878	/* Check limits if it's a normal data segment. */
6879	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6880	{
6881	if ( GCPtrFirst32 > pSel->u32Limit
6882	\|\| GCPtrLast32 > pSel->u32Limit)
6883	{
6884	Log4Func(("Data segment limit exceeded."
6885	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6886	GCPtrLast32, pSel->u32Limit));
6887	if (iSegReg == X86_SREG_SS)
6888	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6889	else
6890	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6891	return VINF_HM_PENDING_XCPT;
6892	}
6893	}
6894	else
6895	{
6896	/* Check limits if it's an expand-down data segment.
6897	Note! The upper boundary is defined by the B bit, not the G bit! */
6898	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6899	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6900	{
6901	Log4Func(("Expand-down data segment limit exceeded."
6902	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6903	GCPtrLast32, pSel->u32Limit));
6904	if (iSegReg == X86_SREG_SS)
6905	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6906	else
6907	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6908	return VINF_HM_PENDING_XCPT;
6909	}
6910	}
6911	}
6912	else
6913	{
6914	/* Check permissions for the code segment. */
6915	if ( enmMemAccess == VMXMEMACCESS_WRITE
6916	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6917	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6918	{
6919	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6920	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6921	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6922	return VINF_HM_PENDING_XCPT;
6923	}
6924
6925	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6926	if ( GCPtrFirst32 > pSel->u32Limit
6927	\|\| GCPtrLast32 > pSel->u32Limit)
6928	{
6929	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6930	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6931	if (iSegReg == X86_SREG_SS)
6932	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6933	else
6934	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6935	return VINF_HM_PENDING_XCPT;
6936	}
6937	}
6938	}
6939	else
6940	{
6941	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6942	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6943	return VINF_HM_PENDING_XCPT;
6944	}
6945
6946	*pGCPtrMem = GCPtrMem;
6947	return VINF_SUCCESS;
6948	}
6949
6950
6951	/**
6952	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6953	* guest attempting to execute a VMX instruction.
6954	*
6955	* @returns Strict VBox status code (i.e. informational status codes too).
6956	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6957	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6958	*
6959	* @param pVCpu The cross context virtual CPU structure.
6960	* @param uExitReason The VM-exit reason.
6961	*
6962	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6963	* @remarks No-long-jump zone!!!
6964	*/
6965	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6966	{
6967	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6968	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6969
6970	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6971	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6972	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6973	{
6974	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6975	hmR0VmxSetPendingXcptUD(pVCpu);
6976	return VINF_HM_PENDING_XCPT;
6977	}
6978
6979	if (uExitReason == VMX_EXIT_VMXON)
6980	{
6981	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6982
6983	/*
6984	* We check CR4.VMXE because it is required to be always set while in VMX operation
6985	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6986	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6987	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6988	*/
6989	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6990	{
6991	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6992	hmR0VmxSetPendingXcptUD(pVCpu);
6993	return VINF_HM_PENDING_XCPT;
6994	}
6995	}
6996	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6997	{
6998	/*
6999	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7000	* (other than VMXON), we need to raise a #UD.
7001	*/
7002	Log4Func(("Not in VMX root mode -> #UD\n"));
7003	hmR0VmxSetPendingXcptUD(pVCpu);
7004	return VINF_HM_PENDING_XCPT;
7005	}
7006
7007	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7008	{
7009	/*
7010	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7011	* the guest hypervisor deal with it.
7012	*/
7013	/** @todo NSTVMX: Trigger a VM-exit */
7014	}
7015
7016	/*
7017	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7018	* (above) takes preceedence over the CPL check.
7019	*/
7020	if (CPUMGetGuestCPL(pVCpu) > 0)
7021	{
7022	Log4Func(("CPL > 0 -> #GP(0)\n"));
7023	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7024	return VINF_HM_PENDING_XCPT;
7025	}
7026
7027	return VINF_SUCCESS;
7028	}
7029	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7030
7031
7032	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7033	{
7034	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7035
7036	/*
7037	* If VT-x marks the segment as unusable, most other bits remain undefined:
7038	* - For CS the L, D and G bits have meaning.
7039	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7040	* - For the remaining data segments no bits are defined.
7041	*
7042	* The present bit and the unusable bit has been observed to be set at the
7043	* same time (the selector was supposed to be invalid as we started executing
7044	* a V8086 interrupt in ring-0).
7045	*
7046	* What should be important for the rest of the VBox code, is that the P bit is
7047	* cleared. Some of the other VBox code recognizes the unusable bit, but
7048	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7049	* safe side here, we'll strip off P and other bits we don't care about. If
7050	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7051	*
7052	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7053	*/
7054	#ifdef VBOX_STRICT
7055	uint32_t const uAttr = pSelReg->Attr.u;
7056	#endif
7057
7058	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7059	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7060	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7061
7062	#ifdef VBOX_STRICT
7063	VMMRZCallRing3Disable(pVCpu);
7064	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7065	# ifdef DEBUG_bird
7066	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7067	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7068	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7069	# endif
7070	VMMRZCallRing3Enable(pVCpu);
7071	NOREF(uAttr);
7072	#endif
7073	RT_NOREF2(pVCpu, idxSel);
7074	}
7075
7076
7077	/**
7078	* Imports a guest segment register from the current VMCS into the guest-CPU
7079	* context.
7080	*
7081	* @returns VBox status code.
7082	* @param pVCpu The cross context virtual CPU structure.
7083	* @param iSegReg The segment register number (X86_SREG_XXX).
7084	*
7085	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7086	* do not log!
7087	*/
7088	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7089	{
7090	Assert(iSegReg < X86_SREG_COUNT);
7091
7092	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7093	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7094	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7095	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7096	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7097	#else
7098	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7099	#endif
7100	uint64_t u64Base;
7101	uint32_t u32Sel, u32Limit, u32Attr;
7102	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7103	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7104	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7105	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7106	if (RT_SUCCESS(rc))
7107	{
7108	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7109	pSelReg->Sel = u32Sel;
7110	pSelReg->ValidSel = u32Sel;
7111	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7112	pSelReg->u32Limit = u32Limit;
7113	pSelReg->u64Base = u64Base;
7114	pSelReg->Attr.u = u32Attr;
7115	if (u32Attr & X86DESCATTR_UNUSABLE)
7116	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7117	}
7118	return rc;
7119	}
7120
7121
7122	/**
7123	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7124	*
7125	* @returns VBox status code.
7126	* @param pVCpu The cross context virtual CPU structure.
7127	*
7128	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7129	* do not log!
7130	*/
7131	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7132	{
7133	uint64_t u64Base;
7134	uint32_t u32Sel, u32Limit, u32Attr;
7135	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7136	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7137	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7138	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7139	if (RT_SUCCESS(rc))
7140	{
7141	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7142	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7143	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7144	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7145	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7146	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7147	if (u32Attr & X86DESCATTR_UNUSABLE)
7148	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7149	}
7150	return rc;
7151	}
7152
7153
7154	/**
7155	* Imports the guest TR from the current VMCS into the guest-CPU context.
7156	*
7157	* @returns VBox status code.
7158	* @param pVCpu The cross context virtual CPU structure.
7159	*
7160	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7161	* do not log!
7162	*/
7163	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7164	{
7165	uint32_t u32Sel, u32Limit, u32Attr;
7166	uint64_t u64Base;
7167	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7168	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7169	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7170	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7171	AssertRCReturn(rc, rc);
7172
7173	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7174	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7175	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7176	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7177	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7178	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7179	/* TR is the only selector that can never be unusable. */
7180	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7181	return VINF_SUCCESS;
7182	}
7183
7184
7185	/**
7186	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7187	*
7188	* @returns VBox status code.
7189	* @param pVCpu The cross context virtual CPU structure.
7190	*
7191	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7192	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7193	* instead!!!
7194	*/
7195	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7196	{
7197	uint64_t u64Val;
7198	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7199	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7200	{
7201	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7202	if (RT_SUCCESS(rc))
7203	{
7204	pCtx->rip = u64Val;
7205	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7206	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7207	}
7208	return rc;
7209	}
7210	return VINF_SUCCESS;
7211	}
7212
7213
7214	/**
7215	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7216	*
7217	* @returns VBox status code.
7218	* @param pVCpu The cross context virtual CPU structure.
7219	* @param pVmcsInfo The VMCS info. object.
7220	*
7221	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7222	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7223	* instead!!!
7224	*/
7225	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7226	{
7227	uint32_t u32Val;
7228	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7229	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7230	{
7231	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7232	if (RT_SUCCESS(rc))
7233	{
7234	pCtx->eflags.u32 = u32Val;
7235
7236	/* Restore eflags for real-on-v86-mode hack. */
7237	if (pVmcsInfo->RealMode.fRealOnV86Active)
7238	{
7239	pCtx->eflags.Bits.u1VM = 0;
7240	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7241	}
7242	}
7243	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7244	return rc;
7245	}
7246	return VINF_SUCCESS;
7247	}
7248
7249
7250	/**
7251	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7252	* context.
7253	*
7254	* @returns VBox status code.
7255	* @param pVCpu The cross context virtual CPU structure.
7256	* @param pVmcsInfo The VMCS info. object.
7257	*
7258	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7259	* do not log!
7260	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7261	* instead!!!
7262	*/
7263	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7264	{
7265	uint32_t u32Val;
7266	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7267	if (RT_SUCCESS(rc))
7268	{
7269	if (!u32Val)
7270	{
7271	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7273
7274	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7275	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7276	}
7277	else
7278	{
7279	/*
7280	* We must import RIP here to set our EM interrupt-inhibited state.
7281	* We also import RFLAGS as our code that evaluates pending interrupts
7282	* before VM-entry requires it.
7283	*/
7284	rc = hmR0VmxImportGuestRip(pVCpu);
7285	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7286	if (RT_SUCCESS(rc))
7287	{
7288	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7289	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7290	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7291	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7292
7293	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7294	{
7295	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7296	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7297	}
7298	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7299	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7300	}
7301	}
7302	}
7303	return rc;
7304	}
7305
7306
7307	/**
7308	* Worker for VMXR0ImportStateOnDemand.
7309	*
7310	* @returns VBox status code.
7311	* @param pVCpu The cross context virtual CPU structure.
7312	* @param pVmcsInfo The VMCS info. object.
7313	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7314	*/
7315	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7316	{
7317	#define VMXLOCAL_BREAK_RC(a_rc) \
7318	if (RT_SUCCESS(a_rc)) \
7319	{ } \
7320	else \
7321	break
7322
7323	int rc = VINF_SUCCESS;
7324	PVM pVM = pVCpu->CTX_SUFF(pVM);
7325	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7326	uint64_t u64Val;
7327	uint32_t u32Val;
7328
7329	/*
7330	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7331	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7332	* neither are other host platforms.
7333	*
7334	* Committing this temporarily as it prevents BSOD.
7335	*/
7336	#ifdef RT_OS_WINDOWS
7337	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7338	return VERR_HM_IPE_1;
7339	#endif
7340
7341	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7342
7343	/*
7344	* We disable interrupts to make the updating of the state and in particular
7345	* the fExtrn modification atomic wrt to preemption hooks.
7346	*/
7347	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7348
7349	fWhat &= pCtx->fExtrn;
7350	if (fWhat)
7351	{
7352	do
7353	{
7354	if (fWhat & CPUMCTX_EXTRN_RIP)
7355	{
7356	rc = hmR0VmxImportGuestRip(pVCpu);
7357	VMXLOCAL_BREAK_RC(rc);
7358	}
7359
7360	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7361	{
7362	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7363	VMXLOCAL_BREAK_RC(rc);
7364	}
7365
7366	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7367	{
7368	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7369	VMXLOCAL_BREAK_RC(rc);
7370	}
7371
7372	if (fWhat & CPUMCTX_EXTRN_RSP)
7373	{
7374	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7375	VMXLOCAL_BREAK_RC(rc);
7376	pCtx->rsp = u64Val;
7377	}
7378
7379	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7380	{
7381	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7382	if (fWhat & CPUMCTX_EXTRN_CS)
7383	{
7384	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7385	rc \|= hmR0VmxImportGuestRip(pVCpu);
7386	if (fRealOnV86Active)
7387	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7388	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7389	}
7390	if (fWhat & CPUMCTX_EXTRN_SS)
7391	{
7392	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7393	if (fRealOnV86Active)
7394	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7395	}
7396	if (fWhat & CPUMCTX_EXTRN_DS)
7397	{
7398	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7399	if (fRealOnV86Active)
7400	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7401	}
7402	if (fWhat & CPUMCTX_EXTRN_ES)
7403	{
7404	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7405	if (fRealOnV86Active)
7406	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7407	}
7408	if (fWhat & CPUMCTX_EXTRN_FS)
7409	{
7410	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7411	if (fRealOnV86Active)
7412	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7413	}
7414	if (fWhat & CPUMCTX_EXTRN_GS)
7415	{
7416	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7417	if (fRealOnV86Active)
7418	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7419	}
7420	VMXLOCAL_BREAK_RC(rc);
7421	}
7422
7423	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7424	{
7425	if (fWhat & CPUMCTX_EXTRN_LDTR)
7426	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7427
7428	if (fWhat & CPUMCTX_EXTRN_GDTR)
7429	{
7430	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7431	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7432	pCtx->gdtr.pGdt = u64Val;
7433	pCtx->gdtr.cbGdt = u32Val;
7434	}
7435
7436	/* Guest IDTR. */
7437	if (fWhat & CPUMCTX_EXTRN_IDTR)
7438	{
7439	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7440	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7441	pCtx->idtr.pIdt = u64Val;
7442	pCtx->idtr.cbIdt = u32Val;
7443	}
7444
7445	/* Guest TR. */
7446	if (fWhat & CPUMCTX_EXTRN_TR)
7447	{
7448	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7449	don't need to import that one. */
7450	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7451	rc \|= hmR0VmxImportGuestTr(pVCpu);
7452	}
7453	VMXLOCAL_BREAK_RC(rc);
7454	}
7455
7456	if (fWhat & CPUMCTX_EXTRN_DR7)
7457	{
7458	if (!pVCpu->hm.s.fUsingHyperDR7)
7459	{
7460	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7461	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7462	VMXLOCAL_BREAK_RC(rc);
7463	pCtx->dr[7] = u32Val;
7464	}
7465	}
7466
7467	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7468	{
7469	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7470	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7471	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7472	pCtx->SysEnter.cs = u32Val;
7473	VMXLOCAL_BREAK_RC(rc);
7474	}
7475
7476	#if HC_ARCH_BITS == 64
7477	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7478	{
7479	if ( pVM->hm.s.fAllow64BitGuests
7480	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7481	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7482	}
7483
7484	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7485	{
7486	if ( pVM->hm.s.fAllow64BitGuests
7487	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7488	{
7489	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7490	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7491	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7492	}
7493	}
7494	#endif
7495
7496	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7497	#if HC_ARCH_BITS == 32
7498	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7499	#endif
7500	)
7501	{
7502	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7503	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7504	Assert(pMsrs);
7505	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7506	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7507	for (uint32_t i = 0; i < cMsrs; i++)
7508	{
7509	uint32_t const idMsr = pMsrs[i].u32Msr;
7510	switch (idMsr)
7511	{
7512	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7513	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7514	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7515	#if HC_ARCH_BITS == 32
7516	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7517	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7518	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7519	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7520	#endif
7521	default:
7522	{
7523	pCtx->fExtrn = 0;
7524	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7525	ASMSetFlags(fEFlags);
7526	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7527	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7528	}
7529	}
7530	}
7531	}
7532
7533	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7534	{
7535	uint64_t u64Shadow;
7536	if (fWhat & CPUMCTX_EXTRN_CR0)
7537	{
7538	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7539	* remove when we drop 32-bit host w/ 64-bit host support, see
7540	* @bugref{9180#c39}. */
7541	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7542	#if HC_ARCH_BITS == 32
7543	uint32_t u32Shadow;
7544	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7545	u64Shadow = u32Shadow;
7546	#else
7547	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7548	#endif
7549	VMXLOCAL_BREAK_RC(rc);
7550	u64Val = u32Val;
7551	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7552	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7553	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7554	CPUMSetGuestCR0(pVCpu, u64Val);
7555	VMMRZCallRing3Enable(pVCpu);
7556	}
7557
7558	if (fWhat & CPUMCTX_EXTRN_CR4)
7559	{
7560	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7561	* remove when we drop 32-bit host w/ 64-bit host support, see
7562	* @bugref{9180#c39}. */
7563	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7564	#if HC_ARCH_BITS == 32
7565	uint32_t u32Shadow;
7566	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7567	u64Shadow = u32Shadow;
7568	#else
7569	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7570	#endif
7571	VMXLOCAL_BREAK_RC(rc);
7572	u64Val = u32Val;
7573	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7574	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7575	pCtx->cr4 = u64Val;
7576	}
7577
7578	if (fWhat & CPUMCTX_EXTRN_CR3)
7579	{
7580	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7581	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7582	\|\| ( pVM->hm.s.fNestedPaging
7583	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7584	{
7585	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7586	VMXLOCAL_BREAK_RC(rc);
7587	if (pCtx->cr3 != u64Val)
7588	{
7589	pCtx->cr3 = u64Val;
7590	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7591	}
7592
7593	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7594	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7595	if (CPUMIsGuestInPAEModeEx(pCtx))
7596	{
7597	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7598	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7599	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7600	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7601	VMXLOCAL_BREAK_RC(rc);
7602	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7603	}
7604	}
7605	}
7606	}
7607	} while (0);
7608
7609	if (RT_SUCCESS(rc))
7610	{
7611	/* Update fExtrn. */
7612	pCtx->fExtrn &= ~fWhat;
7613
7614	/* If everything has been imported, clear the HM keeper bit. */
7615	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7616	{
7617	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7618	Assert(!pCtx->fExtrn);
7619	}
7620	}
7621	}
7622	else
7623	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7624
7625	ASMSetFlags(fEFlags);
7626
7627	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7628
7629	if (RT_SUCCESS(rc))
7630	{ /* likely */ }
7631	else
7632	return rc;
7633
7634	/*
7635	* Honor any pending CR3 updates.
7636	*
7637	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7638	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7639	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7640	*
7641	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7642	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7643	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7644	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7645	*
7646	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7647	*/
7648	if (VMMRZCallRing3IsEnabled(pVCpu))
7649	{
7650	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7651	{
7652	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7653	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7654	}
7655
7656	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7657	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7658
7659	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7660	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7661	}
7662
7663	return VINF_SUCCESS;
7664	#undef VMXLOCAL_BREAK_RC
7665	}
7666
7667
7668	/**
7669	* Saves the guest state from the VMCS into the guest-CPU context.
7670	*
7671	* @returns VBox status code.
7672	* @param pVCpu The cross context virtual CPU structure.
7673	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7674	*/
7675	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7676	{
7677	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7678	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7679	}
7680
7681
7682	/**
7683	* Check per-VM and per-VCPU force flag actions that require us to go back to
7684	* ring-3 for one reason or another.
7685	*
7686	* @returns Strict VBox status code (i.e. informational status codes too)
7687	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7688	* ring-3.
7689	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7690	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7691	* interrupts)
7692	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7693	* all EMTs to be in ring-3.
7694	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7695	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7696	* to the EM loop.
7697	*
7698	* @param pVCpu The cross context virtual CPU structure.
7699	* @param fStepping Whether we are single-stepping the guest using the
7700	* hypervisor debugger.
7701	*/
7702	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7703	{
7704	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7705
7706	/*
7707	* Update pending interrupts into the APIC's IRR.
7708	*/
7709	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7710	APICUpdatePendingInterrupts(pVCpu);
7711
7712	/*
7713	* Anything pending? Should be more likely than not if we're doing a good job.
7714	*/
7715	PVM pVM = pVCpu->CTX_SUFF(pVM);
7716	if ( !fStepping
7717	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7718	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7719	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7720	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7721	return VINF_SUCCESS;
7722
7723	/* Pending PGM C3 sync. */
7724	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7725	{
7726	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7727	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7728	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7729	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7730	if (rcStrict2 != VINF_SUCCESS)
7731	{
7732	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7733	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7734	return rcStrict2;
7735	}
7736	}
7737
7738	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7739	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7740	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7741	{
7742	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7743	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7744	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7745	return rc2;
7746	}
7747
7748	/* Pending VM request packets, such as hardware interrupts. */
7749	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7750	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7751	{
7752	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7753	return VINF_EM_PENDING_REQUEST;
7754	}
7755
7756	/* Pending PGM pool flushes. */
7757	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7758	{
7759	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7760	return VINF_PGM_POOL_FLUSH_PENDING;
7761	}
7762
7763	/* Pending DMA requests. */
7764	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7765	{
7766	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7767	return VINF_EM_RAW_TO_R3;
7768	}
7769
7770	return VINF_SUCCESS;
7771	}
7772
7773
7774	/**
7775	* Converts any TRPM trap into a pending HM event. This is typically used when
7776	* entering from ring-3 (not longjmp returns).
7777	*
7778	* @param pVCpu The cross context virtual CPU structure.
7779	*/
7780	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7781	{
7782	Assert(TRPMHasTrap(pVCpu));
7783	Assert(!pVCpu->hm.s.Event.fPending);
7784
7785	uint8_t uVector;
7786	TRPMEVENT enmTrpmEvent;
7787	RTGCUINT uErrCode;
7788	RTGCUINTPTR GCPtrFaultAddress;
7789	uint8_t cbInstr;
7790
7791	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7792	AssertRC(rc);
7793
7794	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7795	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7796	if (enmTrpmEvent == TRPM_TRAP)
7797	{
7798	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7799	* generated using INT1 (ICEBP). */
7800	switch (uVector)
7801	{
7802	case X86_XCPT_NMI:
7803	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7804	break;
7805
7806	case X86_XCPT_BP:
7807	case X86_XCPT_OF:
7808	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7809	break;
7810
7811	case X86_XCPT_PF:
7812	case X86_XCPT_DF:
7813	case X86_XCPT_TS:
7814	case X86_XCPT_NP:
7815	case X86_XCPT_SS:
7816	case X86_XCPT_GP:
7817	case X86_XCPT_AC:
7818	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7819	RT_FALL_THRU();
7820	default:
7821	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7822	break;
7823	}
7824	}
7825	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7826	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7827	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7828	{
7829	switch (uVector)
7830	{
7831	case X86_XCPT_BP:
7832	case X86_XCPT_OF:
7833	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7834	break;
7835
7836	default:
7837	Assert(uVector == X86_XCPT_DB);
7838	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7839	break;
7840	}
7841	}
7842	else
7843	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7844
7845	rc = TRPMResetTrap(pVCpu);
7846	AssertRC(rc);
7847	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7848	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7849
7850	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7851	}
7852
7853
7854	/**
7855	* Converts the pending HM event into a TRPM trap.
7856	*
7857	* @param pVCpu The cross context virtual CPU structure.
7858	*/
7859	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7860	{
7861	Assert(pVCpu->hm.s.Event.fPending);
7862
7863	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7864	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7865	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7866	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7867
7868	/* If a trap was already pending, we did something wrong! */
7869	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7870
7871	/** @todo Use HMVmxEventToTrpmEventType() later. */
7872	TRPMEVENT enmTrapType;
7873	switch (uVectorType)
7874	{
7875	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7876	enmTrapType = TRPM_HARDWARE_INT;
7877	break;
7878
7879	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7880	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7881	enmTrapType = TRPM_TRAP;
7882	break;
7883
7884	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7885	Assert(uVector == X86_XCPT_DB);
7886	enmTrapType = TRPM_SOFTWARE_INT;
7887	break;
7888
7889	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7890	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7891	enmTrapType = TRPM_SOFTWARE_INT;
7892	break;
7893
7894	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7895	enmTrapType = TRPM_SOFTWARE_INT;
7896	break;
7897
7898	default:
7899	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7900	enmTrapType = TRPM_32BIT_HACK;
7901	break;
7902	}
7903
7904	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7905
7906	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7907	AssertRC(rc);
7908
7909	if (fErrorCodeValid)
7910	TRPMSetErrorCode(pVCpu, uErrorCode);
7911
7912	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7913	&& uVector == X86_XCPT_PF)
7914	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7915	else if (enmTrapType == TRPM_SOFTWARE_INT)
7916	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7917
7918	/* We're now done converting the pending event. */
7919	pVCpu->hm.s.Event.fPending = false;
7920	}
7921
7922
7923	/**
7924	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7925	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7926	*
7927	* @param pVCpu The cross context virtual CPU structure.
7928	* @param pVmcsInfo The VMCS info. object.
7929	*/
7930	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7931	{
7932	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7933	{
7934	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7935	{
7936	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7937	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7938	AssertRC(rc);
7939	}
7940	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7941	}
7942
7943
7944	/**
7945	* Clears the interrupt-window exiting control in the VMCS.
7946	*
7947	* @param pVmcsInfo The VMCS info. object.
7948	*/
7949	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7950	{
7951	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7952	{
7953	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7954	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7955	}
7956	return VINF_SUCCESS;
7957	}
7958
7959
7960	/**
7961	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7962	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7963	*
7964	* @param pVCpu The cross context virtual CPU structure.
7965	* @param pVmcsInfo The VMCS info. object.
7966	*/
7967	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7968	{
7969	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7970	{
7971	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7972	{
7973	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7974	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7975	AssertRC(rc);
7976	Log4Func(("Setup NMI-window exiting\n"));
7977	}
7978	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7979	}
7980
7981
7982	/**
7983	* Clears the NMI-window exiting control in the VMCS.
7984	*
7985	* @param pVmcsInfo The VMCS info. object.
7986	*/
7987	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7988	{
7989	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7990	{
7991	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7992	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7993	}
7994	return VINF_SUCCESS;
7995	}
7996
7997
7998	/**
7999	* Does the necessary state syncing before returning to ring-3 for any reason
8000	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8001	*
8002	* @returns VBox status code.
8003	* @param pVCpu The cross context virtual CPU structure.
8004	* @param fImportState Whether to import the guest state from the VMCS back
8005	* to the guest-CPU context.
8006	*
8007	* @remarks No-long-jmp zone!!!
8008	*/
8009	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8010	{
8011	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8012	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8013
8014	RTCPUID idCpu = RTMpCpuId();
8015	Log4Func(("HostCpuId=%u\n", idCpu));
8016
8017	/*
8018	* !!! IMPORTANT !!!
8019	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8020	*/
8021
8022	/* Save the guest state if necessary. */
8023	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8024	if (fImportState)
8025	{
8026	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8027	AssertRCReturn(rc, rc);
8028	}
8029
8030	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8031	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8032	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8033
8034	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8035	#ifdef VBOX_STRICT
8036	if (CPUMIsHyperDebugStateActive(pVCpu))
8037	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8038	#endif
8039	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8040	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8041	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8042
8043	#if HC_ARCH_BITS == 64
8044	/* Restore host-state bits that VT-x only restores partially. */
8045	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8046	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8047	{
8048	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8049	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8050	}
8051	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8052	#endif
8053
8054	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8055	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8056	{
8057	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8058	if (!fImportState)
8059	{
8060	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8061	AssertRCReturn(rc, rc);
8062	}
8063	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8064	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8065	}
8066	else
8067	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8068
8069	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8070	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8071
8072	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8073	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8074	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8075	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8076	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8077	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8078	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8079	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8080	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8081
8082	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8083
8084	/** @todo This partially defeats the purpose of having preemption hooks.
8085	* The problem is, deregistering the hooks should be moved to a place that
8086	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8087	* context.
8088	*/
8089	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8090	AssertRCReturn(rc, rc);
8091
8092	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8093	NOREF(idCpu);
8094	return VINF_SUCCESS;
8095	}
8096
8097
8098	/**
8099	* Leaves the VT-x session.
8100	*
8101	* @returns VBox status code.
8102	* @param pVCpu The cross context virtual CPU structure.
8103	*
8104	* @remarks No-long-jmp zone!!!
8105	*/
8106	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8107	{
8108	HM_DISABLE_PREEMPT(pVCpu);
8109	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8110	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8111	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8112
8113	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8114	and done this from the VMXR0ThreadCtxCallback(). */
8115	if (!pVCpu->hm.s.fLeaveDone)
8116	{
8117	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8118	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8119	pVCpu->hm.s.fLeaveDone = true;
8120	}
8121	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8122
8123	/*
8124	* !!! IMPORTANT !!!
8125	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8126	*/
8127
8128	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8129	/** @todo Deregistering here means we need to VMCLEAR always
8130	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8131	* for calling VMMR0ThreadCtxHookDisable here! */
8132	VMMR0ThreadCtxHookDisable(pVCpu);
8133
8134	/* Leave HM context. This takes care of local init (term). */
8135	int rc = HMR0LeaveCpu(pVCpu);
8136
8137	HM_RESTORE_PREEMPT();
8138	return rc;
8139	}
8140
8141
8142	/**
8143	* Does the necessary state syncing before doing a longjmp to ring-3.
8144	*
8145	* @returns VBox status code.
8146	* @param pVCpu The cross context virtual CPU structure.
8147	*
8148	* @remarks No-long-jmp zone!!!
8149	*/
8150	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8151	{
8152	return hmR0VmxLeaveSession(pVCpu);
8153	}
8154
8155
8156	/**
8157	* Take necessary actions before going back to ring-3.
8158	*
8159	* An action requires us to go back to ring-3. This function does the necessary
8160	* steps before we can safely return to ring-3. This is not the same as longjmps
8161	* to ring-3, this is voluntary and prepares the guest so it may continue
8162	* executing outside HM (recompiler/IEM).
8163	*
8164	* @returns VBox status code.
8165	* @param pVCpu The cross context virtual CPU structure.
8166	* @param rcExit The reason for exiting to ring-3. Can be
8167	* VINF_VMM_UNKNOWN_RING3_CALL.
8168	*/
8169	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8170	{
8171	Assert(pVCpu);
8172	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8173
8174	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8175	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8176	{
8177	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8178	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8179	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8180	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8181	}
8182
8183	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8184	VMMRZCallRing3Disable(pVCpu);
8185	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8186
8187	/*
8188	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8189	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8190	*
8191	* This is because execution may continue from ring-3 and we would need to inject
8192	* the event from there (hence place it back in TRPM).
8193	*/
8194	if (pVCpu->hm.s.Event.fPending)
8195	{
8196	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8197	Assert(!pVCpu->hm.s.Event.fPending);
8198
8199	/* Clear the events from the VMCS. */
8200	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8201	AssertRCReturn(rc, rc);
8202	}
8203	#ifdef VBOX_STRICT
8204	else
8205	{
8206	/*
8207	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8208	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8209	* occasionally, see @bugref{9180#c42}.
8210	*/
8211	uint32_t uEntryIntInfo;
8212	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8213	AssertRC(rc);
8214	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8215	}
8216	#endif
8217
8218	/*
8219	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8220	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8221	* (e.g. TPR below threshold).
8222	*/
8223	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8224	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8225	AssertRCReturn(rc, rc);
8226
8227	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8228	and if we're injecting an event we should have a TRPM trap pending. */
8229	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8230	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8231	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8232	#endif
8233
8234	/* Save guest state and restore host state bits. */
8235	rc = hmR0VmxLeaveSession(pVCpu);
8236	AssertRCReturn(rc, rc);
8237	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8238
8239	/* Thread-context hooks are unregistered at this point!!! */
8240
8241	/* Sync recompiler state. */
8242	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8243	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8244	\| CPUM_CHANGED_LDTR
8245	\| CPUM_CHANGED_GDTR
8246	\| CPUM_CHANGED_IDTR
8247	\| CPUM_CHANGED_TR
8248	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8249	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8250	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8251	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8252
8253	Assert(!pVCpu->hm.s.fClearTrapFlag);
8254
8255	/* Update the exit-to-ring 3 reason. */
8256	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8257
8258	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8259	if ( rcExit != VINF_EM_RAW_INTERRUPT
8260	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8261	{
8262	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8263	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8264	}
8265
8266	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8267
8268	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8269	VMMRZCallRing3RemoveNotification(pVCpu);
8270	VMMRZCallRing3Enable(pVCpu);
8271
8272	return rc;
8273	}
8274
8275
8276	/**
8277	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8278	* longjump to ring-3 and possibly get preempted.
8279	*
8280	* @returns VBox status code.
8281	* @param pVCpu The cross context virtual CPU structure.
8282	* @param enmOperation The operation causing the ring-3 longjump.
8283	* @param pvUser User argument, currently unused, NULL.
8284	*/
8285	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8286	{
8287	RT_NOREF(pvUser);
8288	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8289	{
8290	/*
8291	* !!! IMPORTANT !!!
8292	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8293	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8294	*/
8295	VMMRZCallRing3RemoveNotification(pVCpu);
8296	VMMRZCallRing3Disable(pVCpu);
8297	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8298	RTThreadPreemptDisable(&PreemptState);
8299
8300	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8301	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8302	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8303	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8304
8305	#if HC_ARCH_BITS == 64
8306	/* Restore host-state bits that VT-x only restores partially. */
8307	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8308	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8309	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8310	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8311	#endif
8312
8313	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8314	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8315	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8316
8317	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8318	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8319	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8320
8321	/* Clear the current VMCS data back to memory. */
8322	hmR0VmxClearVmcs(pVmcsInfo);
8323
8324	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8325	VMMR0ThreadCtxHookDisable(pVCpu);
8326	HMR0LeaveCpu(pVCpu);
8327	RTThreadPreemptRestore(&PreemptState);
8328	return VINF_SUCCESS;
8329	}
8330
8331	Assert(pVCpu);
8332	Assert(pvUser);
8333	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8334	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8335
8336	VMMRZCallRing3Disable(pVCpu);
8337	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8338
8339	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8340
8341	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8342	AssertRCReturn(rc, rc);
8343
8344	VMMRZCallRing3Enable(pVCpu);
8345	return VINF_SUCCESS;
8346	}
8347
8348
8349	/**
8350	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8351	* stack.
8352	*
8353	* @returns Strict VBox status code (i.e. informational status codes too).
8354	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8355	* @param pVCpu The cross context virtual CPU structure.
8356	* @param uValue The value to push to the guest stack.
8357	*/
8358	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8359	{
8360	/*
8361	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8362	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8363	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8364	*/
8365	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8366	if (pCtx->sp == 1)
8367	return VINF_EM_RESET;
8368	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8369	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8370	AssertRC(rc);
8371	return rc;
8372	}
8373
8374
8375	/**
8376	* Injects an event into the guest upon VM-entry by updating the relevant fields
8377	* in the VM-entry area in the VMCS.
8378	*
8379	* @returns Strict VBox status code (i.e. informational status codes too).
8380	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8381	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8382	*
8383	* @param pVCpu The cross context virtual CPU structure.
8384	* @param pVmxTransient The VMX-transient structure.
8385	* @param pEvent The event being injected.
8386	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8387	* This will be updated if necessary. This cannot not
8388	* be NULL.
8389	* @param fStepping Whether we're single-stepping guest execution and
8390	* should return VINF_EM_DBG_STEPPED if the event is
8391	* injected directly (registers modified by us, not by
8392	* hardware on VM-entry).
8393	*/
8394	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8395	uint32_t *pfIntrState)
8396	{
8397	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8398	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8399	Assert(pfIntrState);
8400
8401	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8402	uint32_t u32IntInfo = pEvent->u64IntInfo;
8403	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8404	uint32_t const cbInstr = pEvent->cbInstr;
8405	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8406	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8407	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8408
8409	#ifdef VBOX_STRICT
8410	/*
8411	* Validate the error-code-valid bit for hardware exceptions.
8412	* No error codes for exceptions in real-mode.
8413	*
8414	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8415	*/
8416	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8417	&& !CPUMIsGuestInRealModeEx(pCtx))
8418	{
8419	switch (uVector)
8420	{
8421	case X86_XCPT_PF:
8422	case X86_XCPT_DF:
8423	case X86_XCPT_TS:
8424	case X86_XCPT_NP:
8425	case X86_XCPT_SS:
8426	case X86_XCPT_GP:
8427	case X86_XCPT_AC:
8428	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8429	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8430	RT_FALL_THRU();
8431	default:
8432	break;
8433	}
8434	}
8435
8436	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8437	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8438	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8439	#endif
8440
8441	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8442
8443	/*
8444	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8445	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8446	* interrupt handler in the (real-mode) guest.
8447	*
8448	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8449	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8450	*/
8451	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8452	{
8453	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8454	{
8455	/*
8456	* For CPUs with unrestricted guest execution enabled and with the guest
8457	* in real-mode, we must not set the deliver-error-code bit.
8458	*
8459	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8460	*/
8461	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8462	}
8463	else
8464	{
8465	PVM pVM = pVCpu->CTX_SUFF(pVM);
8466	Assert(PDMVmmDevHeapIsEnabled(pVM));
8467	Assert(pVM->hm.s.vmx.pRealModeTSS);
8468	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8469
8470	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8471	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8472	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8473	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8474	AssertRCReturn(rc2, rc2);
8475
8476	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8477	size_t const cbIdtEntry = sizeof(X86IDTR16);
8478	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8479	{
8480	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8481	if (uVector == X86_XCPT_DF)
8482	return VINF_EM_RESET;
8483
8484	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8485	No error codes for exceptions in real-mode. */
8486	if (uVector == X86_XCPT_GP)
8487	{
8488	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8489	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8490	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8491	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8492	HMEVENT EventXcptDf;
8493	RT_ZERO(EventXcptDf);
8494	EventXcptDf.u64IntInfo = uXcptDfInfo;
8495	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8496	}
8497
8498	/*
8499	* If we're injecting an event with no valid IDT entry, inject a #GP.
8500	* No error codes for exceptions in real-mode.
8501	*
8502	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8503	*/
8504	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8505	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8506	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8507	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8508	HMEVENT EventXcptGp;
8509	RT_ZERO(EventXcptGp);
8510	EventXcptGp.u64IntInfo = uXcptGpInfo;
8511	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8512	}
8513
8514	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8515	uint16_t uGuestIp = pCtx->ip;
8516	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8517	{
8518	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8519	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8520	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8521	}
8522	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8523	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8524
8525	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8526	X86IDTR16 IdtEntry;
8527	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8528	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8529	AssertRCReturn(rc2, rc2);
8530
8531	/* Construct the stack frame for the interrupt/exception handler. */
8532	VBOXSTRICTRC rcStrict;
8533	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8534	if (rcStrict == VINF_SUCCESS)
8535	{
8536	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8537	if (rcStrict == VINF_SUCCESS)
8538	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8539	}
8540
8541	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8542	if (rcStrict == VINF_SUCCESS)
8543	{
8544	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8545	pCtx->rip = IdtEntry.offSel;
8546	pCtx->cs.Sel = IdtEntry.uSel;
8547	pCtx->cs.ValidSel = IdtEntry.uSel;
8548	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8549	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8550	&& uVector == X86_XCPT_PF)
8551	pCtx->cr2 = GCPtrFault;
8552
8553	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8554	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8555	\| HM_CHANGED_GUEST_RSP);
8556
8557	/*
8558	* If we delivered a hardware exception (other than an NMI) and if there was
8559	* block-by-STI in effect, we should clear it.
8560	*/
8561	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8562	{
8563	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8564	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8565	Log4Func(("Clearing inhibition due to STI\n"));
8566	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8567	}
8568
8569	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8570	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8571
8572	/*
8573	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8574	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8575	*/
8576	pVCpu->hm.s.Event.fPending = false;
8577
8578	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8579	if (fStepping)
8580	rcStrict = VINF_EM_DBG_STEPPED;
8581	}
8582	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8583	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8584	return rcStrict;
8585	}
8586	}
8587
8588	/*
8589	* Validate.
8590	*/
8591	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8592	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8593
8594	/*
8595	* Inject the event into the VMCS.
8596	*/
8597	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8598	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8599	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8600	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8601	AssertRCReturn(rc, rc);
8602
8603	/*
8604	* Update guest CR2 if this is a page-fault.
8605	*/
8606	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8607	&& uVector == X86_XCPT_PF)
8608	pCtx->cr2 = GCPtrFault;
8609
8610	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8611	return VINF_SUCCESS;
8612	}
8613
8614
8615	/**
8616	* Evaluates the event to be delivered to the guest and sets it as the pending
8617	* event.
8618	*
8619	* @returns Strict VBox status code (i.e. informational status codes too).
8620	* @param pVCpu The cross context virtual CPU structure.
8621	* @param pVmxTransient The VMX-transient structure.
8622	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8623	*/
8624	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8625	{
8626	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8627	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8628
8629	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8630	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8631	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8632	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8633	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8634
8635	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8636	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8637	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8638	Assert(!TRPMHasTrap(pVCpu));
8639	Assert(pfIntrState);
8640
8641	*pfIntrState = fIntrState;
8642
8643	/*
8644	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8645	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8646	*/
8647	/** @todo SMI. SMIs take priority over NMIs. */
8648	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8649	{
8650	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8651	if ( !pVCpu->hm.s.Event.fPending
8652	&& !fBlockNmi
8653	&& !fBlockSti
8654	&& !fBlockMovSS)
8655	{
8656	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8657	if ( pVmxTransient->fIsNestedGuest
8658	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8659	return IEMExecVmxVmexitNmi(pVCpu);
8660	#endif
8661	hmR0VmxSetPendingXcptNmi(pVCpu);
8662	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8663	Log4Func(("Pending NMI\n"));
8664	}
8665	else
8666	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8667	}
8668	/*
8669	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8670	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8671	*/
8672	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8673	&& !pVCpu->hm.s.fSingleInstruction)
8674	{
8675	Assert(!DBGFIsStepping(pVCpu));
8676	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8677	AssertRCReturn(rc, rc);
8678	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8679	if ( !pVCpu->hm.s.Event.fPending
8680	&& !fBlockInt
8681	&& !fBlockSti
8682	&& !fBlockMovSS)
8683	{
8684	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8685	if ( pVmxTransient->fIsNestedGuest
8686	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8687	{
8688	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8689	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8690	return rcStrict;
8691	}
8692	#endif
8693	uint8_t u8Interrupt;
8694	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8695	if (RT_SUCCESS(rc))
8696	{
8697	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8698	if ( pVmxTransient->fIsNestedGuest
8699	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8700	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8701	{
8702	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8703	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8704	return rcStrict;
8705	}
8706	#endif
8707	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8708	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8709	}
8710	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8711	{
8712	if ( !pVmxTransient->fIsNestedGuest
8713	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8714	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8715	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8716
8717	/*
8718	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8719	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8720	* need to re-set this force-flag here.
8721	*/
8722	}
8723	else
8724	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8725	}
8726	else
8727	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8728	}
8729
8730	return VINF_SUCCESS;
8731	}
8732
8733
8734	/**
8735	* Injects any pending events into the guest if the guest is in a state to
8736	* receive them.
8737	*
8738	* @returns Strict VBox status code (i.e. informational status codes too).
8739	* @param pVCpu The cross context virtual CPU structure.
8740	* @param pVmxTransient The VMX-transient structure.
8741	* @param fIntrState The VT-x guest-interruptibility state.
8742	* @param fStepping Whether we are single-stepping the guest using the
8743	* hypervisor debugger and should return
8744	* VINF_EM_DBG_STEPPED if the event was dispatched
8745	* directly.
8746	*/
8747	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8748	{
8749	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8750	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8751
8752	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8753	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8754
8755	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8756	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8757	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8758	Assert(!TRPMHasTrap(pVCpu));
8759
8760	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8761	if (pVCpu->hm.s.Event.fPending)
8762	{
8763	/*
8764	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8765	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8766	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8767	*
8768	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8769	*/
8770	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8771	#ifdef VBOX_STRICT
8772	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8773	{
8774	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8775	Assert(!fBlockInt);
8776	Assert(!fBlockSti);
8777	Assert(!fBlockMovSS);
8778	}
8779	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8780	{
8781	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8782	Assert(!fBlockSti);
8783	Assert(!fBlockMovSS);
8784	Assert(!fBlockNmi);
8785	}
8786	#endif
8787	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8788	uIntType));
8789
8790	/*
8791	* Inject the event and get any changes to the guest-interruptibility state.
8792	*
8793	* The guest-interruptibility state may need to be updated if we inject the event
8794	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8795	*/
8796	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8797	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8798
8799	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8800	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8801	else
8802	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8803	}
8804
8805	/*
8806	* Update the guest-interruptibility state.
8807	*
8808	* This is required for the real-on-v86 software interrupt injection case above, as well as
8809	* updates to the guest state from ring-3 or IEM/REM.
8810	*/
8811	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8812	AssertRCReturn(rc, rc);
8813
8814	/*
8815	* There's no need to clear the VM-entry interruption-information field here if we're not
8816	* injecting anything. VT-x clears the valid bit on every VM-exit.
8817	*
8818	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8819	*/
8820
8821	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8822	NOREF(fBlockMovSS); NOREF(fBlockSti);
8823	return rcStrict;
8824	}
8825
8826
8827	/**
8828	* Enters the VT-x session.
8829	*
8830	* @returns VBox status code.
8831	* @param pVCpu The cross context virtual CPU structure.
8832	*/
8833	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8834	{
8835	AssertPtr(pVCpu);
8836	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8837	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8838
8839	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8840	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8841	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8842
8843	#ifdef VBOX_STRICT
8844	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8845	RTCCUINTREG uHostCR4 = ASMGetCR4();
8846	if (!(uHostCR4 & X86_CR4_VMXE))
8847	{
8848	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8849	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8850	}
8851	#endif
8852
8853	/*
8854	* Load the appropriate VMCS as the current and active one.
8855	*/
8856	PVMXVMCSINFO pVmcsInfo;
8857	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8858	if (!fInNestedGuestMode)
8859	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8860	else
8861	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8862	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8863	if (RT_SUCCESS(rc))
8864	{
8865	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8866	pVCpu->hm.s.fLeaveDone = false;
8867	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8868
8869	/*
8870	* Do the EMT scheduled L1D flush here if needed.
8871	*/
8872	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8873	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8874	}
8875	return rc;
8876	}
8877
8878
8879	/**
8880	* The thread-context callback (only on platforms which support it).
8881	*
8882	* @param enmEvent The thread-context event.
8883	* @param pVCpu The cross context virtual CPU structure.
8884	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8885	* @thread EMT(pVCpu)
8886	*/
8887	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8888	{
8889	NOREF(fGlobalInit);
8890
8891	switch (enmEvent)
8892	{
8893	case RTTHREADCTXEVENT_OUT:
8894	{
8895	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8896	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8897	VMCPU_ASSERT_EMT(pVCpu);
8898
8899	/* No longjmps (logger flushes, locks) in this fragile context. */
8900	VMMRZCallRing3Disable(pVCpu);
8901	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8902
8903	/* Restore host-state (FPU, debug etc.) */
8904	if (!pVCpu->hm.s.fLeaveDone)
8905	{
8906	/*
8907	* Do -not- import the guest-state here as we might already be in the middle of importing
8908	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8909	*/
8910	hmR0VmxLeave(pVCpu, false /* fImportState */);
8911	pVCpu->hm.s.fLeaveDone = true;
8912	}
8913
8914	/* Leave HM context, takes care of local init (term). */
8915	int rc = HMR0LeaveCpu(pVCpu);
8916	AssertRC(rc);
8917
8918	/* Restore longjmp state. */
8919	VMMRZCallRing3Enable(pVCpu);
8920	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8921	break;
8922	}
8923
8924	case RTTHREADCTXEVENT_IN:
8925	{
8926	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8927	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8928	VMCPU_ASSERT_EMT(pVCpu);
8929
8930	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8931	VMMRZCallRing3Disable(pVCpu);
8932	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8933
8934	/* Initialize the bare minimum state required for HM. This takes care of
8935	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8936	int rc = hmR0EnterCpu(pVCpu);
8937	AssertRC(rc);
8938	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8939	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8940
8941	/* Load the active VMCS as the current one. */
8942	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8943	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8944	AssertRC(rc);
8945	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8946	pVCpu->hm.s.fLeaveDone = false;
8947
8948	/* Do the EMT scheduled L1D flush if needed. */
8949	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8950	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8951
8952	/* Restore longjmp state. */
8953	VMMRZCallRing3Enable(pVCpu);
8954	break;
8955	}
8956
8957	default:
8958	break;
8959	}
8960	}
8961
8962
8963	/**
8964	* Exports the host state into the VMCS host-state area.
8965	* Sets up the VM-exit MSR-load area.
8966	*
8967	* The CPU state will be loaded from these fields on every successful VM-exit.
8968	*
8969	* @returns VBox status code.
8970	* @param pVCpu The cross context virtual CPU structure.
8971	*
8972	* @remarks No-long-jump zone!!!
8973	*/
8974	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8975	{
8976	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8977
8978	int rc = VINF_SUCCESS;
8979	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8980	{
8981	rc = hmR0VmxExportHostControlRegs();
8982	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8983
8984	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8985	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8986
8987	rc = hmR0VmxExportHostMsrs(pVCpu);
8988	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8989
8990	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8991	}
8992	return rc;
8993	}
8994
8995
8996	/**
8997	* Saves the host state in the VMCS host-state.
8998	*
8999	* @returns VBox status code.
9000	* @param pVCpu The cross context virtual CPU structure.
9001	*
9002	* @remarks No-long-jump zone!!!
9003	*/
9004	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9005	{
9006	AssertPtr(pVCpu);
9007	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9008
9009	/*
9010	* Export the host state here while entering HM context.
9011	* When thread-context hooks are used, we might get preempted and have to re-save the host
9012	* state but most of the time we won't be, so do it here before we disable interrupts.
9013	*/
9014	return hmR0VmxExportHostState(pVCpu);
9015	}
9016
9017
9018	/**
9019	* Exports the guest state into the VMCS guest-state area.
9020	*
9021	* The will typically be done before VM-entry when the guest-CPU state and the
9022	* VMCS state may potentially be out of sync.
9023	*
9024	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9025	* VM-entry controls.
9026	* Sets up the appropriate VMX non-root function to execute guest code based on
9027	* the guest CPU mode.
9028	*
9029	* @returns VBox strict status code.
9030	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9031	* without unrestricted guest execution and the VMMDev is not presently
9032	* mapped (e.g. EFI32).
9033	*
9034	* @param pVCpu The cross context virtual CPU structure.
9035	* @param pVmxTransient The VMX-transient structure.
9036	*
9037	* @remarks No-long-jump zone!!!
9038	*/
9039	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9040	{
9041	AssertPtr(pVCpu);
9042	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9043	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9044
9045	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9046
9047	/*
9048	* Determine real-on-v86 mode.
9049	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9050	*/
9051	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9052	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9053	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9054	pVmcsInfo->RealMode. fRealOnV86Active = false;
9055	else
9056	{
9057	Assert(!pVmxTransient->fIsNestedGuest);
9058	pVmcsInfo->RealMode.fRealOnV86Active = true;
9059	}
9060
9061	/*
9062	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9063	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9064	*/
9065	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9066	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9067	* be a need to evaluate this everytime since I'm pretty sure we intercept
9068	* all guest paging mode changes. */
9069	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9070	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9071
9072	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9073	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9074
9075	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9076	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9077
9078	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9079	if (rcStrict == VINF_SUCCESS)
9080	{ /* likely */ }
9081	else
9082	{
9083	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9084	return rcStrict;
9085	}
9086
9087	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9088	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9089
9090	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9091	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9092
9093	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9094	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9095
9096	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9097	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9098
9099	rc = hmR0VmxExportGuestRip(pVCpu);
9100	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9101	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9102	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9103
9104	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9105	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9106	\| HM_CHANGED_GUEST_CR2
9107	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9108	\| HM_CHANGED_GUEST_X87
9109	\| HM_CHANGED_GUEST_SSE_AVX
9110	\| HM_CHANGED_GUEST_OTHER_XSAVE
9111	\| HM_CHANGED_GUEST_XCRx
9112	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9113	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9114	\| HM_CHANGED_GUEST_TSC_AUX
9115	\| HM_CHANGED_GUEST_OTHER_MSRS
9116	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9117	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9118
9119	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9120	return rc;
9121	}
9122
9123
9124	/**
9125	* Exports the state shared between the host and guest into the VMCS.
9126	*
9127	* @param pVCpu The cross context virtual CPU structure.
9128	* @param pVmxTransient The VMX-transient structure.
9129	*
9130	* @remarks No-long-jump zone!!!
9131	*/
9132	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9133	{
9134	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9135	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9136
9137	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9138	{
9139	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9140	AssertRC(rc);
9141	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9142
9143	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9144	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9145	{
9146	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9147	AssertRC(rc);
9148	}
9149	}
9150
9151	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9152	{
9153	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9154	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9155	}
9156
9157	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9158	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9159	}
9160
9161
9162	/**
9163	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9164	*
9165	* @returns Strict VBox status code (i.e. informational status codes too).
9166	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9167	* without unrestricted guest execution and the VMMDev is not presently
9168	* mapped (e.g. EFI32).
9169	*
9170	* @param pVCpu The cross context virtual CPU structure.
9171	* @param pVmxTransient The VMX-transient structure.
9172	*
9173	* @remarks No-long-jump zone!!!
9174	*/
9175	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9176	{
9177	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9178	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9179	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9180
9181	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9182	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9183	#endif
9184
9185	/*
9186	* For many exits it's only RIP that changes and hence try to export it first
9187	* without going through a lot of change flag checks.
9188	*/
9189	VBOXSTRICTRC rcStrict;
9190	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9191	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9192	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9193	{
9194	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9195	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9196	{ /* likely */}
9197	else
9198	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9199	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9200	}
9201	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9202	{
9203	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9204	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9205	{ /* likely */}
9206	else
9207	{
9208	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9209	VBOXSTRICTRC_VAL(rcStrict)));
9210	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9211	return rcStrict;
9212	}
9213	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9214	}
9215	else
9216	rcStrict = VINF_SUCCESS;
9217
9218	#ifdef VBOX_STRICT
9219	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9220	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9221	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9222	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9223	("fCtxChanged=%#RX64\n", fCtxChanged));
9224	#endif
9225	return rcStrict;
9226	}
9227
9228
9229	/**
9230	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9231	* and update error record fields accordingly.
9232	*
9233	* @return VMX_IGS_* return codes.
9234	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9235	* wrong with the guest state.
9236	*
9237	* @param pVCpu The cross context virtual CPU structure.
9238	* @param pVmcsInfo The VMCS info. object.
9239	*
9240	* @remarks This function assumes our cache of the VMCS controls
9241	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9242	*/
9243	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9244	{
9245	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9246	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9247	uError = (err); \
9248	break; \
9249	} else do { } while (0)
9250
9251	int rc;
9252	PVM pVM = pVCpu->CTX_SUFF(pVM);
9253	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9254	uint32_t uError = VMX_IGS_ERROR;
9255	uint32_t u32Val;
9256	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9257
9258	do
9259	{
9260	/*
9261	* CR0.
9262	*/
9263	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9264	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9265	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9266	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9267	if (fUnrestrictedGuest)
9268	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9269
9270	uint32_t u32GuestCr0;
9271	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9272	AssertRCBreak(rc);
9273	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9274	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9275	if ( !fUnrestrictedGuest
9276	&& (u32GuestCr0 & X86_CR0_PG)
9277	&& !(u32GuestCr0 & X86_CR0_PE))
9278	{
9279	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9280	}
9281
9282	/*
9283	* CR4.
9284	*/
9285	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9286	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9287
9288	uint32_t u32GuestCr4;
9289	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9290	AssertRCBreak(rc);
9291	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9292	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9293
9294	/*
9295	* IA32_DEBUGCTL MSR.
9296	*/
9297	uint64_t u64Val;
9298	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9299	AssertRCBreak(rc);
9300	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9301	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9302	{
9303	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9304	}
9305	uint64_t u64DebugCtlMsr = u64Val;
9306
9307	#ifdef VBOX_STRICT
9308	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9309	AssertRCBreak(rc);
9310	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9311	#endif
9312	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9313
9314	/*
9315	* RIP and RFLAGS.
9316	*/
9317	uint32_t u32Eflags;
9318	#if HC_ARCH_BITS == 64
9319	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9320	AssertRCBreak(rc);
9321	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9322	if ( !fLongModeGuest
9323	\|\| !pCtx->cs.Attr.n.u1Long)
9324	{
9325	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9326	}
9327	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9328	* must be identical if the "IA-32e mode guest" VM-entry
9329	* control is 1 and CS.L is 1. No check applies if the
9330	* CPU supports 64 linear-address bits. */
9331
9332	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9333	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9334	AssertRCBreak(rc);
9335	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9336	VMX_IGS_RFLAGS_RESERVED);
9337	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9338	u32Eflags = u64Val;
9339	#else
9340	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9341	AssertRCBreak(rc);
9342	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9343	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9344	#endif
9345
9346	if ( fLongModeGuest
9347	\|\| ( fUnrestrictedGuest
9348	&& !(u32GuestCr0 & X86_CR0_PE)))
9349	{
9350	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9351	}
9352
9353	uint32_t u32EntryInfo;
9354	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9355	AssertRCBreak(rc);
9356	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9357	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9358	{
9359	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9360	}
9361
9362	/*
9363	* 64-bit checks.
9364	*/
9365	#if HC_ARCH_BITS == 64
9366	if (fLongModeGuest)
9367	{
9368	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9369	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9370	}
9371
9372	if ( !fLongModeGuest
9373	&& (u32GuestCr4 & X86_CR4_PCIDE))
9374	{
9375	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9376	}
9377
9378	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9379	* 51:32 beyond the processor's physical-address width are 0. */
9380
9381	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9382	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9383	{
9384	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9385	}
9386
9387	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9388	AssertRCBreak(rc);
9389	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9390
9391	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9392	AssertRCBreak(rc);
9393	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9394	#endif
9395
9396	/*
9397	* PERF_GLOBAL MSR.
9398	*/
9399	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9400	{
9401	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9402	AssertRCBreak(rc);
9403	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9404	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9405	}
9406
9407	/*
9408	* PAT MSR.
9409	*/
9410	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9411	{
9412	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9413	AssertRCBreak(rc);
9414	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9415	for (unsigned i = 0; i < 8; i++)
9416	{
9417	uint8_t u8Val = (u64Val & 0xff);
9418	if ( u8Val != 0 /* UC */
9419	&& u8Val != 1 /* WC */
9420	&& u8Val != 4 /* WT */
9421	&& u8Val != 5 /* WP */
9422	&& u8Val != 6 /* WB */
9423	&& u8Val != 7 /* UC- */)
9424	{
9425	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9426	}
9427	u64Val >>= 8;
9428	}
9429	}
9430
9431	/*
9432	* EFER MSR.
9433	*/
9434	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9435	{
9436	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9437	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9438	AssertRCBreak(rc);
9439	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9440	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9441	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9442	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9443	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9444	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9445	* iemVmxVmentryCheckGuestState(). */
9446	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9447	\|\| !(u32GuestCr0 & X86_CR0_PG)
9448	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9449	VMX_IGS_EFER_LMA_LME_MISMATCH);
9450	}
9451
9452	/*
9453	* Segment registers.
9454	*/
9455	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9456	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9457	if (!(u32Eflags & X86_EFL_VM))
9458	{
9459	/* CS */
9460	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9461	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9462	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9463	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9464	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9465	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9466	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9467	/* CS cannot be loaded with NULL in protected mode. */
9468	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9469	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9470	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9471	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9472	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9473	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9474	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9475	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9476	else
9477	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9478
9479	/* SS */
9480	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9481	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9482	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9483	if ( !(pCtx->cr0 & X86_CR0_PE)
9484	\|\| pCtx->cs.Attr.n.u4Type == 3)
9485	{
9486	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9487	}
9488	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9489	{
9490	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9491	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9492	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9493	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9494	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9495	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9496	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9497	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9498	}
9499
9500	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9501	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9502	{
9503	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9504	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9505	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9506	\|\| pCtx->ds.Attr.n.u4Type > 11
9507	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9508	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9509	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9510	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9511	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9512	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9513	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9514	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9515	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9516	}
9517	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9518	{
9519	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9520	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9521	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9522	\|\| pCtx->es.Attr.n.u4Type > 11
9523	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9524	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9525	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9526	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9527	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9528	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9529	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9530	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9531	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9532	}
9533	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9534	{
9535	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9536	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9537	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9538	\|\| pCtx->fs.Attr.n.u4Type > 11
9539	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9540	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9541	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9542	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9543	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9544	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9545	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9546	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9547	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9548	}
9549	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9550	{
9551	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9552	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9553	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9554	\|\| pCtx->gs.Attr.n.u4Type > 11
9555	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9556	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9557	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9558	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9559	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9560	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9561	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9562	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9563	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9564	}
9565	/* 64-bit capable CPUs. */
9566	#if HC_ARCH_BITS == 64
9567	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9568	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9569	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9570	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9571	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9572	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9573	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9574	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9575	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9576	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9577	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9578	#endif
9579	}
9580	else
9581	{
9582	/* V86 mode checks. */
9583	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9584	if (pVmcsInfo->RealMode.fRealOnV86Active)
9585	{
9586	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9587	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9588	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9589	}
9590	else
9591	{
9592	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9593	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9594	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9595	}
9596
9597	/* CS */
9598	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9599	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9600	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9601	/* SS */
9602	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9603	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9604	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9605	/* DS */
9606	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9607	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9608	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9609	/* ES */
9610	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9611	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9612	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9613	/* FS */
9614	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9615	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9616	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9617	/* GS */
9618	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9619	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9620	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9621	/* 64-bit capable CPUs. */
9622	#if HC_ARCH_BITS == 64
9623	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9624	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9625	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9626	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9627	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9628	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9629	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9630	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9631	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9632	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9633	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9634	#endif
9635	}
9636
9637	/*
9638	* TR.
9639	*/
9640	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9641	/* 64-bit capable CPUs. */
9642	#if HC_ARCH_BITS == 64
9643	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9644	#endif
9645	if (fLongModeGuest)
9646	{
9647	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9648	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9649	}
9650	else
9651	{
9652	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9653	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9654	VMX_IGS_TR_ATTR_TYPE_INVALID);
9655	}
9656	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9657	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9658	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9659	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9660	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9661	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9662	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9663	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9664
9665	/*
9666	* GDTR and IDTR.
9667	*/
9668	#if HC_ARCH_BITS == 64
9669	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9670	AssertRCBreak(rc);
9671	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9672
9673	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9674	AssertRCBreak(rc);
9675	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9676	#endif
9677
9678	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9679	AssertRCBreak(rc);
9680	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9681
9682	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9683	AssertRCBreak(rc);
9684	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9685
9686	/*
9687	* Guest Non-Register State.
9688	*/
9689	/* Activity State. */
9690	uint32_t u32ActivityState;
9691	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9692	AssertRCBreak(rc);
9693	HMVMX_CHECK_BREAK( !u32ActivityState
9694	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9695	VMX_IGS_ACTIVITY_STATE_INVALID);
9696	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9697	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9698	uint32_t u32IntrState;
9699	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9700	AssertRCBreak(rc);
9701	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9702	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9703	{
9704	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9705	}
9706
9707	/** @todo Activity state and injecting interrupts. Left as a todo since we
9708	* currently don't use activity states but ACTIVE. */
9709
9710	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9711	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9712
9713	/* Guest interruptibility-state. */
9714	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9715	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9716	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9717	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9718	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9719	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9720	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9721	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9722	{
9723	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9724	{
9725	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9726	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9727	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9728	}
9729	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9730	{
9731	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9732	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9733	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9734	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9735	}
9736	}
9737	/** @todo Assumes the processor is not in SMM. */
9738	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9739	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9740	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9741	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9742	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9743	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9744	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9745	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9746	{
9747	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9748	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9749	}
9750
9751	/* Pending debug exceptions. */
9752	#if HC_ARCH_BITS == 64
9753	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9754	AssertRCBreak(rc);
9755	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9756	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9757	u32Val = u64Val; /* For pending debug exceptions checks below. */
9758	#else
9759	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9760	AssertRCBreak(rc);
9761	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9762	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9763	#endif
9764
9765	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9766	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9767	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9768	{
9769	if ( (u32Eflags & X86_EFL_TF)
9770	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9771	{
9772	/* Bit 14 is PendingDebug.BS. */
9773	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9774	}
9775	if ( !(u32Eflags & X86_EFL_TF)
9776	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9777	{
9778	/* Bit 14 is PendingDebug.BS. */
9779	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9780	}
9781	}
9782
9783	/* VMCS link pointer. */
9784	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9785	AssertRCBreak(rc);
9786	if (u64Val != UINT64_C(0xffffffffffffffff))
9787	{
9788	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9789	/** @todo Bits beyond the processor's physical-address width MBZ. */
9790	/** @todo 32-bit located in memory referenced by value of this field (as a
9791	* physical address) must contain the processor's VMCS revision ID. */
9792	/** @todo SMM checks. */
9793	}
9794
9795	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9796	* not using nested paging? */
9797	if ( pVM->hm.s.fNestedPaging
9798	&& !fLongModeGuest
9799	&& CPUMIsGuestInPAEModeEx(pCtx))
9800	{
9801	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9802	AssertRCBreak(rc);
9803	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9804
9805	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9806	AssertRCBreak(rc);
9807	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9808
9809	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9810	AssertRCBreak(rc);
9811	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9812
9813	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9814	AssertRCBreak(rc);
9815	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9816	}
9817
9818	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9819	if (uError == VMX_IGS_ERROR)
9820	uError = VMX_IGS_REASON_NOT_FOUND;
9821	} while (0);
9822
9823	pVCpu->hm.s.u32HMError = uError;
9824	return uError;
9825
9826	#undef HMVMX_ERROR_BREAK
9827	#undef HMVMX_CHECK_BREAK
9828	}
9829
9830
9831	/**
9832	* Setup the APIC-access page for virtualizing APIC access.
9833	*
9834	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9835	* this not done as part of exporting guest state, see @bugref{8721}.
9836	*
9837	* @returns VBox status code.
9838	* @param pVCpu The cross context virtual CPU structure.
9839	*/
9840	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9841	{
9842	PVM pVM = pVCpu->CTX_SUFF(pVM);
9843	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9844
9845	Assert(PDMHasApic(pVM));
9846	Assert(u64MsrApicBase);
9847
9848	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9849	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9850
9851	/* Unalias any existing mapping. */
9852	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9853	AssertRCReturn(rc, rc);
9854
9855	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9856	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9857	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9858	AssertRCReturn(rc, rc);
9859
9860	/* Update the per-VCPU cache of the APIC base MSR. */
9861	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9862	return VINF_SUCCESS;
9863	}
9864
9865
9866	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9867	/**
9868	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9869	* nested-guest using hardware-assisted VMX.
9870	*
9871	* @param pVCpu The cross context virtual CPU structure.
9872	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9873	* @param pVmcsInfoGst The guest VMCS info. object.
9874	*/
9875	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9876	{
9877	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9878	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9879	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9880	Assert(pu64MsrBitmapNstGst);
9881	Assert(pu64MsrBitmapGst);
9882	Assert(pu64MsrBitmap);
9883
9884	/*
9885	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9886	* MSR that is intercepted by the guest is also intercepted while executing the
9887	* nested-guest using hardware-assisted VMX.
9888	*/
9889	uint32_t const cbFrag = sizeof(uint64_t);
9890	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9891	for (uint32_t i = 0; i <= cFrags; i++)
9892	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9893	}
9894
9895
9896	/**
9897	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9898	* hardware-assisted VMX execution of the nested-guest.
9899	*
9900	* For a guest, we don't modify these controls once we set up the VMCS.
9901	*
9902	* For nested-guests since the guest hypervisor provides these controls on every
9903	* nested-guest VM-entry and could potentially change them everytime we need to
9904	* merge them before every nested-guest VM-entry.
9905	*
9906	* @returns VBox status code.
9907	* @param pVCpu The cross context virtual CPU structure.
9908	*/
9909	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9910	{
9911	PVM pVM = pVCpu->CTX_SUFF(pVM);
9912	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9913	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9914	Assert(pVmcsNstGst);
9915
9916	/*
9917	* Merge the controls with the requirements of the guest VMCS.
9918	*
9919	* We do not need to validate the nested-guest VMX features specified in the
9920	* nested-guest VMCS with the features supported by the physical CPU as it's
9921	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9922	*
9923	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9924	* guest are derived from the VMX features supported by the physical CPU.
9925	*/
9926
9927	/* Pin-based VM-execution controls. */
9928	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9929
9930	/* Processor-based VM-execution controls. */
9931	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9932	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9933	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9934	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9935	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9936
9937	/* Secondary processor-based VM-execution controls. */
9938	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9939	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9940	\| VMX_PROC_CTLS2_INVPCID
9941	\| VMX_PROC_CTLS2_RDTSCP
9942	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9943	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9944	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9945	\| VMX_PROC_CTLS2_VMFUNC));
9946
9947	/*
9948	* VM-entry controls:
9949	* These controls contains state that depends on the nested-guest state (primarily
9950	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9951	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9952	* properly continue executing the nested-guest if the EFER MSR changes but does not
9953	* cause a nested-guest VM-exits.
9954	*
9955	* VM-exit controls:
9956	* These controls specify the host state on return. We cannot use the controls from
9957	* the nested-hypervisor state as is as it would contain the guest state rather than
9958	* the host state. Since the host state is subject to change (e.g. preemption, trips
9959	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9960	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9961	*
9962	* VM-entry MSR-load:
9963	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9964	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9965	*
9966	* VM-exit MSR-store:
9967	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9968	* context back into the VM-exit MSR-store area.
9969	*
9970	* VM-exit MSR-load areas:
9971	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9972	* we can entirely ignore what the nested-hypervisor wants to load here.
9973	*/
9974
9975	/*
9976	* Exception bitmap.
9977	*
9978	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9979	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9980	* keep the code more flexible if intercepting exceptions become more dynamic in
9981	* the future we do it as part of exporting the nested-guest state.
9982	*/
9983	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9984
9985	/*
9986	* CR0/CR4 guest/host mask.
9987	*
9988	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9989	* must cause VM-exits, so we need to merge them here.
9990	*/
9991	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9992	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9993
9994	/*
9995	* Page-fault error-code mask and match.
9996	*
9997	* Although we require unrestricted guest execution (and thereby nested-paging) for
9998	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9999	* normally intercept #PFs, it might intercept them for debugging purposes.
10000	*
10001	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10002	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10003	*/
10004	uint32_t u32XcptPFMask;
10005	uint32_t u32XcptPFMatch;
10006	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10007	{
10008	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10009	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10010	}
10011	else
10012	{
10013	u32XcptPFMask = 0;
10014	u32XcptPFMatch = 0;
10015	}
10016
10017	/*
10018	* Pause-Loop exiting.
10019	*/
10020	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10021	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10022
10023	/*
10024	* I/O Bitmap.
10025	*
10026	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10027	* always intercept all I/O port accesses.
10028	*/
10029	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10030
10031	/*
10032	* APIC-access page.
10033	*
10034	* The APIC-access page address has already been initialized while setting up the
10035	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10036	* should not be on any consequence to the host or to the guest for that matter, but
10037	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10038	* emulation to keep it simple.
10039	*/
10040
10041	/*
10042	* Virtual-APIC page and TPR threshold.
10043	*
10044	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10045	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10046	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10047	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10048	*/
10049	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10050	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10051	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10052	{
10053	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10054
10055	/*
10056	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10057	* we would fail to obtain a valid host-physical address for its guest-physical
10058	* address.
10059	*
10060	* We currently do not support this scenario. Maybe in the future if there is a
10061	* pressing need we can explore making this particular set of conditions work.
10062	* Right now we just cause a VM-entry failure.
10063	*
10064	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10065	* so should not really failure at the moment.
10066	*/
10067	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10068	}
10069	else
10070	{
10071	/*
10072	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10073	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10074	* be taken care of by EPT/shadow paging.
10075	*/
10076	if (pVM->hm.s.fAllow64BitGuests)
10077	{
10078	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10079	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10080	}
10081	}
10082
10083	/*
10084	* Validate basic assumptions.
10085	*/
10086	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10087	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10088	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10089	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10090
10091	/*
10092	* Commit it to the nested-guest VMCS.
10093	*/
10094	int rc = VINF_SUCCESS;
10095	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10096	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10097	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10098	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10099	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10100	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10101	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10102	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10103	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10104	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10105	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10106	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10107	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10108	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10109	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10110	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10111	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10112	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10113	{
10114	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10115	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10116	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10117	}
10118	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10119	{
10120	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10121	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10122	}
10123	AssertRCReturn(rc, rc);
10124
10125	/*
10126	* Update the nested-guest VMCS cache.
10127	*/
10128	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10129	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10130	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10131	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10132	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10133	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10134	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10135	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10136	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10137
10138	/*
10139	* MSR bitmap.
10140	*
10141	* The MSR bitmap address has already been initialized while setting up the
10142	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10143	*/
10144	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10145	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10146
10147	return VINF_SUCCESS;
10148	}
10149	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10150
10151
10152	/**
10153	* Does the preparations before executing guest code in VT-x.
10154	*
10155	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10156	* recompiler/IEM. We must be cautious what we do here regarding committing
10157	* guest-state information into the VMCS assuming we assuredly execute the
10158	* guest in VT-x mode.
10159	*
10160	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10161	* the common-state (TRPM/forceflags), we must undo those changes so that the
10162	* recompiler/IEM can (and should) use them when it resumes guest execution.
10163	* Otherwise such operations must be done when we can no longer exit to ring-3.
10164	*
10165	* @returns Strict VBox status code (i.e. informational status codes too).
10166	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10167	* have been disabled.
10168	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10169	* double-fault into the guest.
10170	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10171	* dispatched directly.
10172	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10173	*
10174	* @param pVCpu The cross context virtual CPU structure.
10175	* @param pVmxTransient The VMX-transient structure.
10176	* @param fStepping Whether we are single-stepping the guest in the
10177	* hypervisor debugger. Makes us ignore some of the reasons
10178	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10179	* if event dispatching took place.
10180	*/
10181	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10182	{
10183	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10184
10185	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10186	if (pVmxTransient->fIsNestedGuest)
10187	{
10188	RT_NOREF2(pVCpu, fStepping);
10189	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10190	return VINF_EM_RESCHEDULE_REM;
10191	}
10192	#endif
10193
10194	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10195	PGMRZDynMapFlushAutoSet(pVCpu);
10196	#endif
10197
10198	/*
10199	* Check and process force flag actions, some of which might require us to go back to ring-3.
10200	*/
10201	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10202	if (rcStrict == VINF_SUCCESS)
10203	{ /* FFs don't get set all the time. */ }
10204	else
10205	return rcStrict;
10206
10207	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10208	/*
10209	* Switch to the nested-guest VMCS as we may have transitioned into executing
10210	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10211	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10212	*
10213	* We do this as late as possible to minimize (though not completely remove)
10214	* clearing/loading VMCS again due to premature trips to ring-3 above.
10215	*/
10216	if (pVmxTransient->fIsNestedGuest)
10217	{
10218	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10219	{
10220	/*
10221	* Ensure we have synced everything from the guest VMCS and also flag that
10222	* that we need to export the full (nested) guest-CPU context to the
10223	* nested-guest VMCS.
10224	*/
10225	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10226	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10227
10228	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10229	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10230	if (RT_LIKELY(rc == VINF_SUCCESS))
10231	{
10232	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10233	ASMSetFlags(fEFlags);
10234	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10235
10236	/*
10237	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10238	* flag that we need to update the host MSR values there. Even if we decide
10239	* in the future to share the VM-exit MSR-store area page with the guest,
10240	* if its content differs, we would have to update the host MSRs anyway.
10241	*/
10242	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10243	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10244	}
10245	else
10246	{
10247	ASMSetFlags(fEFlags);
10248	return rc;
10249	}
10250	}
10251
10252	/*
10253	* Merge guest VMCS controls with the nested-guest VMCS controls.
10254	*
10255	* Even if we have not executed the guest prior to this (e.g. when resuming
10256	* from a saved state), we should be okay with merging controls as we
10257	* initialize the guest VMCS controls as part of VM setup phase.
10258	*/
10259	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10260	{
10261	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10262	AssertRCReturn(rc, rc);
10263	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10264	}
10265	}
10266	#endif
10267
10268	/*
10269	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10270	* We look at the guest VMCS control here as we always set it when supported by
10271	* the physical CPU. Looking at the nested-guest control here would not be
10272	* possible because they are not merged yet.
10273	*/
10274	PVM pVM = pVCpu->CTX_SUFF(pVM);
10275	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10276	Assert(pVmcsInfo);
10277	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10278	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10279	&& PDMHasApic(pVM))
10280	{
10281	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10282	AssertRCReturn(rc, rc);
10283	}
10284
10285	/*
10286	* Evaluate events to be injected into the guest.
10287	*
10288	* Events in TRPM can be injected without inspecting the guest state.
10289	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10290	* guest to cause a VM-exit the next time they are ready to receive the event.
10291	*/
10292	if (TRPMHasTrap(pVCpu))
10293	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10294
10295	uint32_t fIntrState;
10296	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10297
10298	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10299	/*
10300	* While evaluating pending events if something failed (unlikely) or if we were
10301	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10302	*/
10303	if ( rcStrict != VINF_SUCCESS
10304	\|\| ( pVmxTransient->fIsNestedGuest
10305	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10306	return rcStrict;
10307	#endif
10308
10309	/*
10310	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10311	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10312	* also result in triple-faulting the VM.
10313	*
10314	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10315	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10316	*/
10317	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10318	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10319	{ /* likely */ }
10320	else
10321	{
10322	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10323	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10324	return rcStrict;
10325	}
10326
10327	/*
10328	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10329	* import CR3 themselves. We will need to update them here, as even as late as the above
10330	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10331	* the below force flags to be set.
10332	*/
10333	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10334	{
10335	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10336	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10337	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10338	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10339	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10340	}
10341	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10342	{
10343	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10344	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10345	}
10346
10347	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10348	/* Paranoia. */
10349	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10350	#endif
10351
10352	/*
10353	* No longjmps to ring-3 from this point on!!!
10354	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10355	* This also disables flushing of the R0-logger instance (if any).
10356	*/
10357	VMMRZCallRing3Disable(pVCpu);
10358
10359	/*
10360	* Export the guest state bits.
10361	*
10362	* We cannot perform longjmps while loading the guest state because we do not preserve the
10363	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10364	* CPU migration.
10365	*
10366	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10367	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10368	*/
10369	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10370	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10371	{ /* likely */ }
10372	else
10373	{
10374	VMMRZCallRing3Enable(pVCpu);
10375	return rcStrict;
10376	}
10377
10378	/*
10379	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10380	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10381	* preemption disabled for a while. Since this is purely to aid the
10382	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10383	* disable interrupt on NT.
10384	*
10385	* We need to check for force-flags that could've possible been altered since we last
10386	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10387	* see @bugref{6398}).
10388	*
10389	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10390	* to ring-3 before executing guest code.
10391	*/
10392	pVmxTransient->fEFlags = ASMIntDisableFlags();
10393
10394	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10395	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10396	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10397	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10398	{
10399	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10400	{
10401	pVCpu->hm.s.Event.fPending = false;
10402
10403	/*
10404	* We've injected any pending events. This is really the point of no return (to ring-3).
10405	*
10406	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10407	* returns from this function, so don't enable them here.
10408	*/
10409	return VINF_SUCCESS;
10410	}
10411
10412	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10413	rcStrict = VINF_EM_RAW_INTERRUPT;
10414	}
10415	else
10416	{
10417	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10418	rcStrict = VINF_EM_RAW_TO_R3;
10419	}
10420
10421	ASMSetFlags(pVmxTransient->fEFlags);
10422	VMMRZCallRing3Enable(pVCpu);
10423
10424	return rcStrict;
10425	}
10426
10427
10428	/**
10429	* Final preparations before executing guest code using hardware-assisted VMX.
10430	*
10431	* We can no longer get preempted to a different host CPU and there are no returns
10432	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10433	* failures), this function is not intended to fail sans unrecoverable hardware
10434	* errors.
10435	*
10436	* @param pVCpu The cross context virtual CPU structure.
10437	* @param pVmxTransient The VMX-transient structure.
10438	*
10439	* @remarks Called with preemption disabled.
10440	* @remarks No-long-jump zone!!!
10441	*/
10442	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10443	{
10444	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10445	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10446	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10447	Assert(!pVCpu->hm.s.Event.fPending);
10448
10449	/*
10450	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10451	*/
10452	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10453	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10454
10455	PVM pVM = pVCpu->CTX_SUFF(pVM);
10456	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10457
10458	if (!CPUMIsGuestFPUStateActive(pVCpu))
10459	{
10460	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10461	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10462	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10463	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10464	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10465	}
10466
10467	/*
10468	* Re-save the host state bits as we may've been preempted (only happens when
10469	* thread-context hooks are used or when the VM start function changes).
10470	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10471	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10472	* see @bugref{8432}.
10473	*
10474	* This may also happen when switching to/from a nested-guest VMCS without leaving
10475	* ring-0.
10476	*/
10477	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10478	{
10479	int rc = hmR0VmxExportHostState(pVCpu);
10480	AssertRC(rc);
10481	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10482	}
10483	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10484
10485	/*
10486	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10487	*/
10488	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10489	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10490	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10491
10492	/*
10493	* Store status of the shared guest/host debug state at the time of VM-entry.
10494	*/
10495	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10496	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10497	{
10498	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10499	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10500	}
10501	else
10502	#endif
10503	{
10504	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10505	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10506	}
10507
10508	/*
10509	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10510	* more than one conditional check. The post-run side of our code shall determine
10511	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10512	*/
10513	if (pVmcsInfo->pbVirtApic)
10514	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10515
10516	/*
10517	* Update the host MSRs values in the VM-exit MSR-load area.
10518	*/
10519	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10520	{
10521	if (pVmcsInfo->cExitMsrLoad > 0)
10522	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10523	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10524	}
10525
10526	/*
10527	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10528	* VMX-preemption timer based on the next virtual sync clock deadline.
10529	*/
10530	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10531	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10532	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10533	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10534	{
10535	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10536	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10537	}
10538
10539	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10540	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10541	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10542	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10543
10544	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10545
10546	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10547	as we're about to start executing the guest . */
10548
10549	/*
10550	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10551	*
10552	* This is done this late as updating the TSC offsetting/preemption timer above
10553	* figures out if we can skip intercepting RDTSCP by calculating the number of
10554	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10555	*/
10556	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10557	{
10558	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10559	{
10560	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10561	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10562	* it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done.
10563	*/
10564	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10565	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10566	AssertRC(rc);
10567	}
10568	else
10569	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10570	}
10571
10572	#ifdef VBOX_STRICT
10573	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10574	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10575	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10576	#endif
10577
10578	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10579	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10580	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10581	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10582	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10583	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10584	#endif
10585	}
10586
10587
10588	/**
10589	* First C routine invoked after running guest code using hardware-assisted VMX.
10590	*
10591	* @param pVCpu The cross context virtual CPU structure.
10592	* @param pVmxTransient The VMX-transient structure.
10593	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10594	*
10595	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10596	*
10597	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10598	* unconditionally when it is safe to do so.
10599	*/
10600	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10601	{
10602	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10603
10604	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10605	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10606	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10607	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10608	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10609	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10610
10611	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10612	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10613	{
10614	uint64_t uGstTsc;
10615	if (!pVmxTransient->fIsNestedGuest)
10616	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10617	else
10618	{
10619	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10620	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10621	}
10622	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10623	}
10624
10625	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10626	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10627	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10628
10629	#if HC_ARCH_BITS == 64
10630	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10631	#endif
10632	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10633	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10634	and we need to leave it alone here. */
10635	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10636	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10637	#else
10638	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10639	#endif
10640	#ifdef VBOX_STRICT
10641	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10642	#endif
10643	Assert(!ASMIntAreEnabled());
10644	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10645	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10646
10647	/*
10648	* Save the basic VM-exit reason and check if the VM-entry failed.
10649	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10650	*/
10651	uint32_t uExitReason;
10652	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10653	AssertRC(rc);
10654	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10655	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10656
10657	/*
10658	* Check if VMLAUNCH/VMRESUME succeeded.
10659	* If this failed, we cause a guru meditation and cease further execution.
10660	*/
10661	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10662	{
10663	/*
10664	* Update the VM-exit history array here even if the VM-entry failed due to:
10665	* - Invalid guest state.
10666	* - MSR loading.
10667	* - Machine-check event.
10668	*
10669	* In any of the above cases we will still have a "valid" VM-exit reason
10670	* despite @a fVMEntryFailed being false.
10671	*
10672	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10673	*
10674	* Note! We don't have CS or RIP at this point. Will probably address that later
10675	* by amending the history entry added here.
10676	*/
10677	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10678	UINT64_MAX, uHostTsc);
10679
10680	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10681	{
10682	VMMRZCallRing3Enable(pVCpu);
10683
10684	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10685	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10686
10687	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10688	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10689	AssertRC(rc);
10690	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10691	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10692	AssertRC(rc);
10693	#else
10694	/*
10695	* Import the guest-interruptibility state always as we need it while evaluating
10696	* injecting events on re-entry.
10697	*
10698	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10699	* checking for real-mode while exporting the state because all bits that cause
10700	* mode changes wrt CR0 are intercepted.
10701	*/
10702	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10703	AssertRC(rc);
10704	#endif
10705
10706	/*
10707	* Sync the TPR shadow with our APIC state.
10708	*/
10709	if ( !pVmxTransient->fIsNestedGuest
10710	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10711	{
10712	Assert(pVmcsInfo->pbVirtApic);
10713	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10714	{
10715	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10716	AssertRC(rc);
10717	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10718	}
10719	}
10720
10721	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10722	return;
10723	}
10724	}
10725	else
10726	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10727
10728	VMMRZCallRing3Enable(pVCpu);
10729	}
10730
10731
10732	/**
10733	* Runs the guest code using hardware-assisted VMX the normal way.
10734	*
10735	* @returns VBox status code.
10736	* @param pVCpu The cross context virtual CPU structure.
10737	* @param pcLoops Pointer to the number of executed loops.
10738	*/
10739	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10740	{
10741	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10742	Assert(pcLoops);
10743	Assert(*pcLoops <= cMaxResumeLoops);
10744
10745	VMXTRANSIENT VmxTransient;
10746	RT_ZERO(VmxTransient);
10747	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10748
10749	/* Paranoia. */
10750	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10751	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10752
10753	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10754	for (;;)
10755	{
10756	Assert(!HMR0SuspendPending());
10757	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10758	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10759
10760	/*
10761	* Preparatory work for running nested-guest code, this may force us to
10762	* return to ring-3.
10763	*
10764	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10765	*/
10766	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10767	if (rcStrict != VINF_SUCCESS)
10768	break;
10769
10770	/* Interrupts are disabled at this point! */
10771	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10772	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10773	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10774	/* Interrupts are re-enabled at this point! */
10775
10776	/*
10777	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10778	*/
10779	if (RT_SUCCESS(rcRun))
10780	{ /* very likely */ }
10781	else
10782	{
10783	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10784	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10785	return rcRun;
10786	}
10787
10788	/*
10789	* Profile the VM-exit.
10790	*/
10791	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10793	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10794	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10795	HMVMX_START_EXIT_DISPATCH_PROF();
10796
10797	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10798
10799	/*
10800	* Handle the VM-exit.
10801	*/
10802	#ifdef HMVMX_USE_FUNCTION_TABLE
10803	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10804	#else
10805	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10806	#endif
10807	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10808	if (rcStrict == VINF_SUCCESS)
10809	{
10810	if (++(*pcLoops) <= cMaxResumeLoops)
10811	continue;
10812	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10813	rcStrict = VINF_EM_RAW_INTERRUPT;
10814	}
10815	break;
10816	}
10817
10818	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10819	return rcStrict;
10820	}
10821
10822	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10823	/**
10824	* Runs the nested-guest code using hardware-assisted VMX.
10825	*
10826	* @returns VBox status code.
10827	* @param pVCpu The cross context virtual CPU structure.
10828	* @param pcLoops Pointer to the number of executed loops.
10829	*
10830	* @sa hmR0VmxRunGuestCodeNormal().
10831	*/
10832	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10833	{
10834	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10835	Assert(pcLoops);
10836	Assert(*pcLoops <= cMaxResumeLoops);
10837
10838	VMXTRANSIENT VmxTransient;
10839	RT_ZERO(VmxTransient);
10840	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10841	VmxTransient.fIsNestedGuest = true;
10842
10843	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10844	for (;;)
10845	{
10846	Assert(!HMR0SuspendPending());
10847	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10848	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10849
10850	/*
10851	* Preparatory work for running guest code, this may force us to
10852	* return to ring-3.
10853	*
10854	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10855	*/
10856	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10857	if (rcStrict != VINF_SUCCESS)
10858	break;
10859
10860	/* Interrupts are disabled at this point! */
10861	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10862	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10863	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10864	/* Interrupts are re-enabled at this point! */
10865
10866	/*
10867	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10868	*/
10869	if (RT_SUCCESS(rcRun))
10870	{ /* very likely */ }
10871	else
10872	{
10873	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10874	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10875	return rcRun;
10876	}
10877
10878	/*
10879	* Profile the VM-exit.
10880	*/
10881	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10883	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10884	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10885	HMVMX_START_EXIT_DISPATCH_PROF();
10886
10887	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10888
10889	/*
10890	* Handle the VM-exit.
10891	*/
10892	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10893	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10894	if ( rcStrict == VINF_SUCCESS
10895	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10896	{
10897	if (++(*pcLoops) <= cMaxResumeLoops)
10898	continue;
10899	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10900	rcStrict = VINF_EM_RAW_INTERRUPT;
10901	}
10902	break;
10903	}
10904
10905	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10906	return rcStrict;
10907	}
10908	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10909
10910
10911	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10912	* probes.
10913	*
10914	* The following few functions and associated structure contains the bloat
10915	* necessary for providing detailed debug events and dtrace probes as well as
10916	* reliable host side single stepping. This works on the principle of
10917	* "subclassing" the normal execution loop and workers. We replace the loop
10918	* method completely and override selected helpers to add necessary adjustments
10919	* to their core operation.
10920	*
10921	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10922	* any performance for debug and analysis features.
10923	*
10924	* @{
10925	*/
10926
10927	/**
10928	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10929	* the debug run loop.
10930	*/
10931	typedef struct VMXRUNDBGSTATE
10932	{
10933	/** The RIP we started executing at. This is for detecting that we stepped. */
10934	uint64_t uRipStart;
10935	/** The CS we started executing with. */
10936	uint16_t uCsStart;
10937
10938	/** Whether we've actually modified the 1st execution control field. */
10939	bool fModifiedProcCtls : 1;
10940	/** Whether we've actually modified the 2nd execution control field. */
10941	bool fModifiedProcCtls2 : 1;
10942	/** Whether we've actually modified the exception bitmap. */
10943	bool fModifiedXcptBitmap : 1;
10944
10945	/** We desire the modified the CR0 mask to be cleared. */
10946	bool fClearCr0Mask : 1;
10947	/** We desire the modified the CR4 mask to be cleared. */
10948	bool fClearCr4Mask : 1;
10949	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10950	uint32_t fCpe1Extra;
10951	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10952	uint32_t fCpe1Unwanted;
10953	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10954	uint32_t fCpe2Extra;
10955	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10956	uint32_t bmXcptExtra;
10957	/** The sequence number of the Dtrace provider settings the state was
10958	* configured against. */
10959	uint32_t uDtraceSettingsSeqNo;
10960	/** VM-exits to check (one bit per VM-exit). */
10961	uint32_t bmExitsToCheck[3];
10962
10963	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10964	uint32_t fProcCtlsInitial;
10965	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10966	uint32_t fProcCtls2Initial;
10967	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10968	uint32_t bmXcptInitial;
10969	} VMXRUNDBGSTATE;
10970	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10971	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10972
10973
10974	/**
10975	* Initializes the VMXRUNDBGSTATE structure.
10976	*
10977	* @param pVCpu The cross context virtual CPU structure of the
10978	* calling EMT.
10979	* @param pVmxTransient The VMX-transient structure.
10980	* @param pDbgState The debug state to initialize.
10981	*/
10982	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10983	{
10984	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10985	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10986
10987	pDbgState->fModifiedProcCtls = false;
10988	pDbgState->fModifiedProcCtls2 = false;
10989	pDbgState->fModifiedXcptBitmap = false;
10990	pDbgState->fClearCr0Mask = false;
10991	pDbgState->fClearCr4Mask = false;
10992	pDbgState->fCpe1Extra = 0;
10993	pDbgState->fCpe1Unwanted = 0;
10994	pDbgState->fCpe2Extra = 0;
10995	pDbgState->bmXcptExtra = 0;
10996	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10997	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10998	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10999	}
11000
11001
11002	/**
11003	* Updates the VMSC fields with changes requested by @a pDbgState.
11004	*
11005	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11006	* immediately before executing guest code, i.e. when interrupts are disabled.
11007	* We don't check status codes here as we cannot easily assert or return in the
11008	* latter case.
11009	*
11010	* @param pVCpu The cross context virtual CPU structure.
11011	* @param pVmxTransient The VMX-transient structure.
11012	* @param pDbgState The debug state.
11013	*/
11014	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11015	{
11016	/*
11017	* Ensure desired flags in VMCS control fields are set.
11018	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11019	*
11020	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11021	* there should be no stale data in pCtx at this point.
11022	*/
11023	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11024	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11025	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11026	{
11027	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11028	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11029	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11030	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11031	pDbgState->fModifiedProcCtls = true;
11032	}
11033
11034	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11035	{
11036	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11037	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11038	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11039	pDbgState->fModifiedProcCtls2 = true;
11040	}
11041
11042	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11043	{
11044	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11045	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11046	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11047	pDbgState->fModifiedXcptBitmap = true;
11048	}
11049
11050	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11051	{
11052	pVmcsInfo->u64Cr0Mask = 0;
11053	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11054	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11055	}
11056
11057	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11058	{
11059	pVmcsInfo->u64Cr4Mask = 0;
11060	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11061	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11062	}
11063
11064	NOREF(pVCpu);
11065	}
11066
11067
11068	/**
11069	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11070	* re-entry next time around.
11071	*
11072	* @returns Strict VBox status code (i.e. informational status codes too).
11073	* @param pVCpu The cross context virtual CPU structure.
11074	* @param pVmxTransient The VMX-transient structure.
11075	* @param pDbgState The debug state.
11076	* @param rcStrict The return code from executing the guest using single
11077	* stepping.
11078	*/
11079	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11080	VBOXSTRICTRC rcStrict)
11081	{
11082	/*
11083	* Restore VM-exit control settings as we may not reenter this function the
11084	* next time around.
11085	*/
11086	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11087
11088	/* We reload the initial value, trigger what we can of recalculations the
11089	next time around. From the looks of things, that's all that's required atm. */
11090	if (pDbgState->fModifiedProcCtls)
11091	{
11092	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11093	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11094	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11095	AssertRCReturn(rc2, rc2);
11096	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11097	}
11098
11099	/* We're currently the only ones messing with this one, so just restore the
11100	cached value and reload the field. */
11101	if ( pDbgState->fModifiedProcCtls2
11102	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11103	{
11104	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11105	AssertRCReturn(rc2, rc2);
11106	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11107	}
11108
11109	/* If we've modified the exception bitmap, we restore it and trigger
11110	reloading and partial recalculation the next time around. */
11111	if (pDbgState->fModifiedXcptBitmap)
11112	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11113
11114	return rcStrict;
11115	}
11116
11117
11118	/**
11119	* Configures VM-exit controls for current DBGF and DTrace settings.
11120	*
11121	* This updates @a pDbgState and the VMCS execution control fields to reflect
11122	* the necessary VM-exits demanded by DBGF and DTrace.
11123	*
11124	* @param pVCpu The cross context virtual CPU structure.
11125	* @param pVmxTransient The VMX-transient structure. May update
11126	* fUpdatedTscOffsettingAndPreemptTimer.
11127	* @param pDbgState The debug state.
11128	*/
11129	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11130	{
11131	/*
11132	* Take down the dtrace serial number so we can spot changes.
11133	*/
11134	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11135	ASMCompilerBarrier();
11136
11137	/*
11138	* We'll rebuild most of the middle block of data members (holding the
11139	* current settings) as we go along here, so start by clearing it all.
11140	*/
11141	pDbgState->bmXcptExtra = 0;
11142	pDbgState->fCpe1Extra = 0;
11143	pDbgState->fCpe1Unwanted = 0;
11144	pDbgState->fCpe2Extra = 0;
11145	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11146	pDbgState->bmExitsToCheck[i] = 0;
11147
11148	/*
11149	* Software interrupts (INT XXh) - no idea how to trigger these...
11150	*/
11151	PVM pVM = pVCpu->CTX_SUFF(pVM);
11152	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11153	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11154	{
11155	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11156	}
11157
11158	/*
11159	* INT3 breakpoints - triggered by #BP exceptions.
11160	*/
11161	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11162	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11163
11164	/*
11165	* Exception bitmap and XCPT events+probes.
11166	*/
11167	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11168	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11169	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11170
11171	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11172	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11173	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11174	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11175	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11176	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11177	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11178	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11179	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11180	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11181	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11182	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11183	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11184	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11185	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11186	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11187	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11188	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11189
11190	if (pDbgState->bmXcptExtra)
11191	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11192
11193	/*
11194	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11195	*
11196	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11197	* So, when adding/changing/removing please don't forget to update it.
11198	*
11199	* Some of the macros are picking up local variables to save horizontal space,
11200	* (being able to see it in a table is the lesser evil here).
11201	*/
11202	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11203	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11204	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11205	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11206	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11207	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11208	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11209	} else do { } while (0)
11210	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11211	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11212	{ \
11213	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11214	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11215	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11216	} else do { } while (0)
11217	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11218	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11219	{ \
11220	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11221	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11222	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11223	} else do { } while (0)
11224	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11225	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11226	{ \
11227	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11228	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11229	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11230	} else do { } while (0)
11231
11232	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11233	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11234	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11235	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11236	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11237
11238	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11240	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11242	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11243	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11244	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11246	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11247	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11248	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11249	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11250	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11251	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11252	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11253	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11254	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11255	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11256	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11257	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11258	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11259	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11260	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11261	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11262	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11263	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11264	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11265	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11266	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11267	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11268	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11269	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11270	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11271	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11272	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11273	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11274
11275	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11276	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11277	{
11278	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11279	\| CPUMCTX_EXTRN_APIC_TPR);
11280	AssertRC(rc);
11281
11282	#if 0 /** @todo fix me */
11283	pDbgState->fClearCr0Mask = true;
11284	pDbgState->fClearCr4Mask = true;
11285	#endif
11286	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11287	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11288	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11289	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11290	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11291	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11292	require clearing here and in the loop if we start using it. */
11293	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11294	}
11295	else
11296	{
11297	if (pDbgState->fClearCr0Mask)
11298	{
11299	pDbgState->fClearCr0Mask = false;
11300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11301	}
11302	if (pDbgState->fClearCr4Mask)
11303	{
11304	pDbgState->fClearCr4Mask = false;
11305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11306	}
11307	}
11308	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11309	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11310
11311	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11312	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11313	{
11314	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11315	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11316	}
11317	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11318	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11319
11320	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11321	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11322	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11323	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11324	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11325	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11326	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11327	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11328	#if 0 /** @todo too slow, fix handler. */
11329	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11330	#endif
11331	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11332
11333	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11334	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11335	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11336	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11337	{
11338	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11339	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11340	}
11341	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11342	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11343	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11344	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11345
11346	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11347	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11348	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11349	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11350	{
11351	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11352	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11353	}
11354	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11355	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11356	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11357	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11358
11359	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11360	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11361	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11362	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11363	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11364	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11365	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11366	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11367	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11368	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11369	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11370	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11371	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11372	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11373	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11374	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11375	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11376	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11377	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11378	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11379	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11380	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11381
11382	#undef IS_EITHER_ENABLED
11383	#undef SET_ONLY_XBM_IF_EITHER_EN
11384	#undef SET_CPE1_XBM_IF_EITHER_EN
11385	#undef SET_CPEU_XBM_IF_EITHER_EN
11386	#undef SET_CPE2_XBM_IF_EITHER_EN
11387
11388	/*
11389	* Sanitize the control stuff.
11390	*/
11391	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11392	if (pDbgState->fCpe2Extra)
11393	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11394	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11395	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11396	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11397	{
11398	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11399	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11400	}
11401
11402	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11403	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11404	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11405	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11406	}
11407
11408
11409	/**
11410	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11411	* appropriate.
11412	*
11413	* The caller has checked the VM-exit against the
11414	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11415	* already, so we don't have to do that either.
11416	*
11417	* @returns Strict VBox status code (i.e. informational status codes too).
11418	* @param pVCpu The cross context virtual CPU structure.
11419	* @param pVmxTransient The VMX-transient structure.
11420	* @param uExitReason The VM-exit reason.
11421	*
11422	* @remarks The name of this function is displayed by dtrace, so keep it short
11423	* and to the point. No longer than 33 chars long, please.
11424	*/
11425	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11426	{
11427	/*
11428	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11429	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11430	*
11431	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11432	* does. Must add/change/remove both places. Same ordering, please.
11433	*
11434	* Added/removed events must also be reflected in the next section
11435	* where we dispatch dtrace events.
11436	*/
11437	bool fDtrace1 = false;
11438	bool fDtrace2 = false;
11439	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11440	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11441	uint32_t uEventArg = 0;
11442	#define SET_EXIT(a_EventSubName) \
11443	do { \
11444	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11445	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11446	} while (0)
11447	#define SET_BOTH(a_EventSubName) \
11448	do { \
11449	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11450	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11451	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11452	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11453	} while (0)
11454	switch (uExitReason)
11455	{
11456	case VMX_EXIT_MTF:
11457	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11458
11459	case VMX_EXIT_XCPT_OR_NMI:
11460	{
11461	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11462	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11463	{
11464	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11465	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11466	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11467	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11468	{
11469	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11470	{
11471	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11472	uEventArg = pVmxTransient->uExitIntErrorCode;
11473	}
11474	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11475	switch (enmEvent1)
11476	{
11477	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11478	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11479	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11480	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11481	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11482	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11483	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11484	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11485	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11486	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11487	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11488	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11489	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11490	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11491	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11492	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11493	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11494	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11495	default: break;
11496	}
11497	}
11498	else
11499	AssertFailed();
11500	break;
11501
11502	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11503	uEventArg = idxVector;
11504	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11505	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11506	break;
11507	}
11508	break;
11509	}
11510
11511	case VMX_EXIT_TRIPLE_FAULT:
11512	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11513	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11514	break;
11515	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11516	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11517	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11518	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11519	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11520
11521	/* Instruction specific VM-exits: */
11522	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11523	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11524	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11525	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11526	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11527	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11528	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11529	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11530	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11531	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11532	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11533	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11534	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11535	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11536	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11537	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11538	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11539	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11540	case VMX_EXIT_MOV_CRX:
11541	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11542	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11543	SET_BOTH(CRX_READ);
11544	else
11545	SET_BOTH(CRX_WRITE);
11546	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11547	break;
11548	case VMX_EXIT_MOV_DRX:
11549	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11550	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11551	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11552	SET_BOTH(DRX_READ);
11553	else
11554	SET_BOTH(DRX_WRITE);
11555	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11556	break;
11557	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11558	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11559	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11560	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11561	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11562	case VMX_EXIT_GDTR_IDTR_ACCESS:
11563	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11564	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11565	{
11566	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11567	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11568	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11569	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11570	}
11571	break;
11572
11573	case VMX_EXIT_LDTR_TR_ACCESS:
11574	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11575	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11576	{
11577	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11578	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11579	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11580	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11581	}
11582	break;
11583
11584	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11585	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11586	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11587	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11588	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11589	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11590	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11591	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11592	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11593	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11594	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11595
11596	/* Events that aren't relevant at this point. */
11597	case VMX_EXIT_EXT_INT:
11598	case VMX_EXIT_INT_WINDOW:
11599	case VMX_EXIT_NMI_WINDOW:
11600	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11601	case VMX_EXIT_PREEMPT_TIMER:
11602	case VMX_EXIT_IO_INSTR:
11603	break;
11604
11605	/* Errors and unexpected events. */
11606	case VMX_EXIT_INIT_SIGNAL:
11607	case VMX_EXIT_SIPI:
11608	case VMX_EXIT_IO_SMI:
11609	case VMX_EXIT_SMI:
11610	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11611	case VMX_EXIT_ERR_MSR_LOAD:
11612	case VMX_EXIT_ERR_MACHINE_CHECK:
11613	break;
11614
11615	default:
11616	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11617	break;
11618	}
11619	#undef SET_BOTH
11620	#undef SET_EXIT
11621
11622	/*
11623	* Dtrace tracepoints go first. We do them here at once so we don't
11624	* have to copy the guest state saving and stuff a few dozen times.
11625	* Down side is that we've got to repeat the switch, though this time
11626	* we use enmEvent since the probes are a subset of what DBGF does.
11627	*/
11628	if (fDtrace1 \|\| fDtrace2)
11629	{
11630	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11631	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11632	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11633	switch (enmEvent1)
11634	{
11635	/** @todo consider which extra parameters would be helpful for each probe. */
11636	case DBGFEVENT_END: break;
11637	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11638	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11639	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11640	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11641	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11642	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11643	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11644	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11645	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11646	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11647	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11648	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11649	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11650	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11651	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11652	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11653	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11654	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11655	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11656	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11657	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11658	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11659	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11660	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11661	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11662	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11665	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11666	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11667	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11668	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11669	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11670	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11671	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11678	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11679	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11680	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11681	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11682	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11683	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11684	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11685	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11686	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11687	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11688	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11689	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11690	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11691	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11692	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11693	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11694	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11695	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11696	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11697	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11698	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11699	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11700	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11701	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11702	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11703	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11704	}
11705	switch (enmEvent2)
11706	{
11707	/** @todo consider which extra parameters would be helpful for each probe. */
11708	case DBGFEVENT_END: break;
11709	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11710	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11711	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11712	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11713	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11714	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11715	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11716	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11719	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11720	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11721	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11722	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11723	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11724	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11725	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11736	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11737	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11738	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11739	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11740	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11741	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11742	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11743	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11744	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11745	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11746	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11747	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11748	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11749	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11750	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11751	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11752	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11753	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11754	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11755	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11756	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11757	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11758	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11759	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11760	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11761	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11762	}
11763	}
11764
11765	/*
11766	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11767	* the DBGF call will do a full check).
11768	*
11769	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11770	* Note! If we have to events, we prioritize the first, i.e. the instruction
11771	* one, in order to avoid event nesting.
11772	*/
11773	PVM pVM = pVCpu->CTX_SUFF(pVM);
11774	if ( enmEvent1 != DBGFEVENT_END
11775	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11776	{
11777	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11778	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11779	if (rcStrict != VINF_SUCCESS)
11780	return rcStrict;
11781	}
11782	else if ( enmEvent2 != DBGFEVENT_END
11783	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11784	{
11785	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11786	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11787	if (rcStrict != VINF_SUCCESS)
11788	return rcStrict;
11789	}
11790
11791	return VINF_SUCCESS;
11792	}
11793
11794
11795	/**
11796	* Single-stepping VM-exit filtering.
11797	*
11798	* This is preprocessing the VM-exits and deciding whether we've gotten far
11799	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11800	* handling is performed.
11801	*
11802	* @returns Strict VBox status code (i.e. informational status codes too).
11803	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11804	* @param pVmxTransient The VMX-transient structure.
11805	* @param pDbgState The debug state.
11806	*/
11807	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11808	{
11809	/*
11810	* Expensive (saves context) generic dtrace VM-exit probe.
11811	*/
11812	uint32_t const uExitReason = pVmxTransient->uExitReason;
11813	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11814	{ /* more likely */ }
11815	else
11816	{
11817	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11818	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11819	AssertRC(rc);
11820	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11821	}
11822
11823	/*
11824	* Check for host NMI, just to get that out of the way.
11825	*/
11826	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11827	{ /* normally likely */ }
11828	else
11829	{
11830	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11831	AssertRCReturn(rc2, rc2);
11832	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11833	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11834	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11835	}
11836
11837	/*
11838	* Check for single stepping event if we're stepping.
11839	*/
11840	if (pVCpu->hm.s.fSingleInstruction)
11841	{
11842	switch (uExitReason)
11843	{
11844	case VMX_EXIT_MTF:
11845	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11846
11847	/* Various events: */
11848	case VMX_EXIT_XCPT_OR_NMI:
11849	case VMX_EXIT_EXT_INT:
11850	case VMX_EXIT_TRIPLE_FAULT:
11851	case VMX_EXIT_INT_WINDOW:
11852	case VMX_EXIT_NMI_WINDOW:
11853	case VMX_EXIT_TASK_SWITCH:
11854	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11855	case VMX_EXIT_APIC_ACCESS:
11856	case VMX_EXIT_EPT_VIOLATION:
11857	case VMX_EXIT_EPT_MISCONFIG:
11858	case VMX_EXIT_PREEMPT_TIMER:
11859
11860	/* Instruction specific VM-exits: */
11861	case VMX_EXIT_CPUID:
11862	case VMX_EXIT_GETSEC:
11863	case VMX_EXIT_HLT:
11864	case VMX_EXIT_INVD:
11865	case VMX_EXIT_INVLPG:
11866	case VMX_EXIT_RDPMC:
11867	case VMX_EXIT_RDTSC:
11868	case VMX_EXIT_RSM:
11869	case VMX_EXIT_VMCALL:
11870	case VMX_EXIT_VMCLEAR:
11871	case VMX_EXIT_VMLAUNCH:
11872	case VMX_EXIT_VMPTRLD:
11873	case VMX_EXIT_VMPTRST:
11874	case VMX_EXIT_VMREAD:
11875	case VMX_EXIT_VMRESUME:
11876	case VMX_EXIT_VMWRITE:
11877	case VMX_EXIT_VMXOFF:
11878	case VMX_EXIT_VMXON:
11879	case VMX_EXIT_MOV_CRX:
11880	case VMX_EXIT_MOV_DRX:
11881	case VMX_EXIT_IO_INSTR:
11882	case VMX_EXIT_RDMSR:
11883	case VMX_EXIT_WRMSR:
11884	case VMX_EXIT_MWAIT:
11885	case VMX_EXIT_MONITOR:
11886	case VMX_EXIT_PAUSE:
11887	case VMX_EXIT_GDTR_IDTR_ACCESS:
11888	case VMX_EXIT_LDTR_TR_ACCESS:
11889	case VMX_EXIT_INVEPT:
11890	case VMX_EXIT_RDTSCP:
11891	case VMX_EXIT_INVVPID:
11892	case VMX_EXIT_WBINVD:
11893	case VMX_EXIT_XSETBV:
11894	case VMX_EXIT_RDRAND:
11895	case VMX_EXIT_INVPCID:
11896	case VMX_EXIT_VMFUNC:
11897	case VMX_EXIT_RDSEED:
11898	case VMX_EXIT_XSAVES:
11899	case VMX_EXIT_XRSTORS:
11900	{
11901	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11902	AssertRCReturn(rc, rc);
11903	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11904	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11905	return VINF_EM_DBG_STEPPED;
11906	break;
11907	}
11908
11909	/* Errors and unexpected events: */
11910	case VMX_EXIT_INIT_SIGNAL:
11911	case VMX_EXIT_SIPI:
11912	case VMX_EXIT_IO_SMI:
11913	case VMX_EXIT_SMI:
11914	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11915	case VMX_EXIT_ERR_MSR_LOAD:
11916	case VMX_EXIT_ERR_MACHINE_CHECK:
11917	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11918	break;
11919
11920	default:
11921	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11922	break;
11923	}
11924	}
11925
11926	/*
11927	* Check for debugger event breakpoints and dtrace probes.
11928	*/
11929	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11930	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11931	{
11932	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11933	if (rcStrict != VINF_SUCCESS)
11934	return rcStrict;
11935	}
11936
11937	/*
11938	* Normal processing.
11939	*/
11940	#ifdef HMVMX_USE_FUNCTION_TABLE
11941	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11942	#else
11943	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11944	#endif
11945	}
11946
11947
11948	/**
11949	* Single steps guest code using hardware-assisted VMX.
11950	*
11951	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11952	* but single-stepping through the hypervisor debugger.
11953	*
11954	* @returns Strict VBox status code (i.e. informational status codes too).
11955	* @param pVCpu The cross context virtual CPU structure.
11956	* @param pcLoops Pointer to the number of executed loops.
11957	*
11958	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11959	*/
11960	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11961	{
11962	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11963	Assert(pcLoops);
11964	Assert(*pcLoops <= cMaxResumeLoops);
11965
11966	VMXTRANSIENT VmxTransient;
11967	RT_ZERO(VmxTransient);
11968	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11969
11970	/* Set HMCPU indicators. */
11971	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11972	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11973	pVCpu->hm.s.fDebugWantRdTscExit = false;
11974	pVCpu->hm.s.fUsingDebugLoop = true;
11975
11976	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11977	VMXRUNDBGSTATE DbgState;
11978	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11979	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11980
11981	/*
11982	* The loop.
11983	*/
11984	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11985	for (;;)
11986	{
11987	Assert(!HMR0SuspendPending());
11988	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11989	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11990	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11991
11992	/* Set up VM-execution controls the next two can respond to. */
11993	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11994
11995	/*
11996	* Preparatory work for running guest code, this may force us to
11997	* return to ring-3.
11998	*
11999	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12000	*/
12001	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12002	if (rcStrict != VINF_SUCCESS)
12003	break;
12004
12005	/* Interrupts are disabled at this point! */
12006	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12007
12008	/* Override any obnoxious code in the above two calls. */
12009	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12010
12011	/*
12012	* Finally execute the guest.
12013	*/
12014	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12015
12016	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12017	/* Interrupts are re-enabled at this point! */
12018
12019	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12020	if (RT_SUCCESS(rcRun))
12021	{ /* very likely */ }
12022	else
12023	{
12024	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12025	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12026	return rcRun;
12027	}
12028
12029	/* Profile the VM-exit. */
12030	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12031	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12032	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12033	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12034	HMVMX_START_EXIT_DISPATCH_PROF();
12035
12036	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12037
12038	/*
12039	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12040	*/
12041	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12042	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12043	if (rcStrict != VINF_SUCCESS)
12044	break;
12045	if (++(*pcLoops) > cMaxResumeLoops)
12046	{
12047	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12048	rcStrict = VINF_EM_RAW_INTERRUPT;
12049	break;
12050	}
12051
12052	/*
12053	* Stepping: Did the RIP change, if so, consider it a single step.
12054	* Otherwise, make sure one of the TFs gets set.
12055	*/
12056	if (fStepping)
12057	{
12058	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12059	AssertRC(rc);
12060	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12061	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12062	{
12063	rcStrict = VINF_EM_DBG_STEPPED;
12064	break;
12065	}
12066	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12067	}
12068
12069	/*
12070	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12071	*/
12072	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12073	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12074	}
12075
12076	/*
12077	* Clear the X86_EFL_TF if necessary.
12078	*/
12079	if (pVCpu->hm.s.fClearTrapFlag)
12080	{
12081	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12082	AssertRC(rc);
12083	pVCpu->hm.s.fClearTrapFlag = false;
12084	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12085	}
12086	/** @todo there seems to be issues with the resume flag when the monitor trap
12087	* flag is pending without being used. Seen early in bios init when
12088	* accessing APIC page in protected mode. */
12089
12090	/*
12091	* Restore VM-exit control settings as we may not re-enter this function the
12092	* next time around.
12093	*/
12094	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12095
12096	/* Restore HMCPU indicators. */
12097	pVCpu->hm.s.fUsingDebugLoop = false;
12098	pVCpu->hm.s.fDebugWantRdTscExit = false;
12099	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12100
12101	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12102	return rcStrict;
12103	}
12104
12105
12106	/** @} */
12107
12108
12109	/**
12110	* Checks if any expensive dtrace probes are enabled and we should go to the
12111	* debug loop.
12112	*
12113	* @returns true if we should use debug loop, false if not.
12114	*/
12115	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12116	{
12117	/* It's probably faster to OR the raw 32-bit counter variables together.
12118	Since the variables are in an array and the probes are next to one
12119	another (more or less), we have good locality. So, better read
12120	eight-nine cache lines ever time and only have one conditional, than
12121	128+ conditionals, right? */
12122	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12123	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12124	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12125	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12126	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12127	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12128	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12129	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12130	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12131	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12132	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12133	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12134	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12135	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12136	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12137	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12138	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12139	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12140	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12141	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12142	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12143	) != 0
12144	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12165	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12166	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12167	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12168	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12169	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12170	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12171	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12172	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12173	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12174	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12175	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12176	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12177	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12178	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12179	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12180	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12181	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12182	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12183	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12184	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12185	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12186	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12187	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12188	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12189	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12190	) != 0
12191	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12217	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12218	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12219	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12220	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12221	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12222	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12223	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12224	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12225	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12226	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12227	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12228	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12229	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12230	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12231	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12232	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12233	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12234	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12235	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12236	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12237	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12238	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12239	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12240	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12241	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12242	) != 0;
12243	}
12244
12245
12246	/**
12247	* Runs the guest using hardware-assisted VMX.
12248	*
12249	* @returns Strict VBox status code (i.e. informational status codes too).
12250	* @param pVCpu The cross context virtual CPU structure.
12251	*/
12252	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12253	{
12254	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12255	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12256	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12257	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12258
12259	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12260
12261	VBOXSTRICTRC rcStrict;
12262	uint32_t cLoops = 0;
12263	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12264	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12265	#else
12266	bool const fInNestedGuestMode = false;
12267	#endif
12268	if (!fInNestedGuestMode)
12269	{
12270	if ( !pVCpu->hm.s.fUseDebugLoop
12271	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12272	&& !DBGFIsStepping(pVCpu)
12273	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12274	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12275	else
12276	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12277	}
12278	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12279	else
12280	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12281
12282	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12283	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12284	#endif
12285
12286	if (rcStrict == VERR_EM_INTERPRETER)
12287	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12288	else if (rcStrict == VINF_EM_RESET)
12289	rcStrict = VINF_EM_TRIPLE_FAULT;
12290
12291	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12292	if (RT_FAILURE(rc2))
12293	{
12294	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12295	rcStrict = rc2;
12296	}
12297	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12298	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12299	return rcStrict;
12300	}
12301
12302
12303	#ifndef HMVMX_USE_FUNCTION_TABLE
12304	/**
12305	* Handles a guest VM-exit from hardware-assisted VMX execution.
12306	*
12307	* @returns Strict VBox status code (i.e. informational status codes too).
12308	* @param pVCpu The cross context virtual CPU structure.
12309	* @param pVmxTransient The VMX-transient structure.
12310	*/
12311	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12312	{
12313	#ifdef DEBUG_ramshankar
12314	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12315	do { \
12316	if (a_fSave != 0) \
12317	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12318	VBOXSTRICTRC rcStrict = a_CallExpr; \
12319	if (a_fSave != 0) \
12320	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12321	return rcStrict; \
12322	} while (0)
12323	#else
12324	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12325	#endif
12326	uint32_t const rcReason = pVmxTransient->uExitReason;
12327	switch (rcReason)
12328	{
12329	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12330	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12331	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12332	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12333	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12334	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12335	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12336	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12337	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12338	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12339	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12340	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12341	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12342	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12343	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12344	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12345	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12346	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12347	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12348	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12349	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12350	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12351	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12352	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12353	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12354	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12355	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12356	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12357	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12358	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12359	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12360	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12361	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12362	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12363	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12364	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12365	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12366	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12367	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12368	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12369	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12370	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12371	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12372	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12373	#else
12374	case VMX_EXIT_VMCLEAR:
12375	case VMX_EXIT_VMLAUNCH:
12376	case VMX_EXIT_VMPTRLD:
12377	case VMX_EXIT_VMPTRST:
12378	case VMX_EXIT_VMREAD:
12379	case VMX_EXIT_VMRESUME:
12380	case VMX_EXIT_VMWRITE:
12381	case VMX_EXIT_VMXOFF:
12382	case VMX_EXIT_VMXON:
12383	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12384	#endif
12385
12386	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12387	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12388	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12389	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12390	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12391	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12392	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12393	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12394	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12395
12396	case VMX_EXIT_INVEPT:
12397	case VMX_EXIT_INVVPID:
12398	case VMX_EXIT_VMFUNC:
12399	case VMX_EXIT_XSAVES:
12400	case VMX_EXIT_XRSTORS:
12401	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12402
12403	case VMX_EXIT_ENCLS:
12404	case VMX_EXIT_RDSEED:
12405	case VMX_EXIT_PML_FULL:
12406	default:
12407	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12408	}
12409	#undef VMEXIT_CALL_RET
12410	}
12411	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12412
12413
12414	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12415	/**
12416	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12417	*
12418	* @returns Strict VBox status code (i.e. informational status codes too).
12419	* @param pVCpu The cross context virtual CPU structure.
12420	* @param pVmxTransient The VMX-transient structure.
12421	*/
12422	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12423	{
12424	uint32_t const rcReason = pVmxTransient->uExitReason;
12425	switch (rcReason)
12426	{
12427	case VMX_EXIT_EPT_MISCONFIG:
12428	case VMX_EXIT_EPT_VIOLATION:
12429	case VMX_EXIT_IO_INSTR:
12430	case VMX_EXIT_CPUID:
12431	case VMX_EXIT_RDTSC:
12432	case VMX_EXIT_RDTSCP:
12433	case VMX_EXIT_APIC_ACCESS:
12434	case VMX_EXIT_XCPT_OR_NMI:
12435	case VMX_EXIT_MOV_CRX:
12436	case VMX_EXIT_EXT_INT:
12437	case VMX_EXIT_INT_WINDOW:
12438	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12439	case VMX_EXIT_MWAIT:
12440	case VMX_EXIT_MONITOR:
12441	case VMX_EXIT_TASK_SWITCH:
12442	case VMX_EXIT_PREEMPT_TIMER:
12443	case VMX_EXIT_RDMSR:
12444	case VMX_EXIT_WRMSR:
12445	case VMX_EXIT_VMCALL:
12446	case VMX_EXIT_MOV_DRX:
12447	case VMX_EXIT_HLT:
12448	case VMX_EXIT_INVD:
12449	case VMX_EXIT_INVLPG:
12450	case VMX_EXIT_RSM:
12451	case VMX_EXIT_MTF:
12452	case VMX_EXIT_PAUSE:
12453	case VMX_EXIT_GDTR_IDTR_ACCESS:
12454	case VMX_EXIT_LDTR_TR_ACCESS:
12455	case VMX_EXIT_WBINVD:
12456	case VMX_EXIT_XSETBV:
12457	case VMX_EXIT_RDRAND:
12458	case VMX_EXIT_INVPCID:
12459	case VMX_EXIT_GETSEC:
12460	case VMX_EXIT_RDPMC:
12461	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12462	case VMX_EXIT_VMCLEAR:
12463	case VMX_EXIT_VMLAUNCH:
12464	case VMX_EXIT_VMPTRLD:
12465	case VMX_EXIT_VMPTRST:
12466	case VMX_EXIT_VMREAD:
12467	case VMX_EXIT_VMRESUME:
12468	case VMX_EXIT_VMWRITE:
12469	case VMX_EXIT_VMXOFF:
12470	case VMX_EXIT_VMXON:
12471	#endif
12472	case VMX_EXIT_TRIPLE_FAULT:
12473	case VMX_EXIT_NMI_WINDOW:
12474	case VMX_EXIT_INIT_SIGNAL:
12475	case VMX_EXIT_SIPI:
12476	case VMX_EXIT_IO_SMI:
12477	case VMX_EXIT_SMI:
12478	case VMX_EXIT_ERR_MSR_LOAD:
12479	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12480	case VMX_EXIT_ERR_MACHINE_CHECK:
12481
12482	case VMX_EXIT_INVEPT:
12483	case VMX_EXIT_INVVPID:
12484	case VMX_EXIT_VMFUNC:
12485	case VMX_EXIT_XSAVES:
12486	case VMX_EXIT_XRSTORS:
12487
12488	case VMX_EXIT_ENCLS:
12489	case VMX_EXIT_RDSEED:
12490	case VMX_EXIT_PML_FULL:
12491	default:
12492	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12493	}
12494	#undef VMEXIT_CALL_RET
12495	}
12496	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12497
12498
12499	#ifdef VBOX_STRICT
12500	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12501	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12502	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12503
12504	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12505	do { \
12506	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12507	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12508	} while (0)
12509
12510	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12511	do { \
12512	AssertPtr((a_pVCpu)); \
12513	AssertPtr((a_pVmxTransient)); \
12514	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12515	Assert((a_pVmxTransient)->pVmcsInfo); \
12516	Assert(ASMIntAreEnabled()); \
12517	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12518	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12519	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12520	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12521	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12522	HMVMX_ASSERT_PREEMPT_CPUID(); \
12523	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12524	} while (0)
12525
12526	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12527	do { \
12528	Log4Func(("\n")); \
12529	} while (0)
12530	#else
12531	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12532	do { \
12533	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12534	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12535	} while (0)
12536	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12537	#endif
12538
12539
12540	/**
12541	* Advances the guest RIP by the specified number of bytes.
12542	*
12543	* @param pVCpu The cross context virtual CPU structure.
12544	* @param cbInstr Number of bytes to advance the RIP by.
12545	*
12546	* @remarks No-long-jump zone!!!
12547	*/
12548	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12549	{
12550	/* Advance the RIP. */
12551	pVCpu->cpum.GstCtx.rip += cbInstr;
12552	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12553
12554	/* Update interrupt inhibition. */
12555	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12556	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12557	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12558	}
12559
12560
12561	/**
12562	* Advances the guest RIP after reading it from the VMCS.
12563	*
12564	* @returns VBox status code, no informational status codes.
12565	* @param pVCpu The cross context virtual CPU structure.
12566	* @param pVmxTransient The VMX-transient structure.
12567	*
12568	* @remarks No-long-jump zone!!!
12569	*/
12570	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12571	{
12572	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12573	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12574	AssertRCReturn(rc, rc);
12575
12576	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12577	return VINF_SUCCESS;
12578	}
12579
12580
12581	/**
12582	* Handle a condition that occurred while delivering an event through the guest
12583	* IDT.
12584	*
12585	* @returns Strict VBox status code (i.e. informational status codes too).
12586	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12587	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12588	* to continue execution of the guest which will delivery the \#DF.
12589	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12590	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12591	*
12592	* @param pVCpu The cross context virtual CPU structure.
12593	* @param pVmxTransient The VMX-transient structure.
12594	*
12595	* @remarks No-long-jump zone!!!
12596	*/
12597	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12598	{
12599	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12600
12601	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12602	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12603	AssertRCReturn(rc2, rc2);
12604
12605	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12606	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12607	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12608	{
12609	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12610	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12611
12612	/*
12613	* If the event was a software interrupt (generated with INT n) or a software exception
12614	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12615	* can handle the VM-exit and continue guest execution which will re-execute the
12616	* instruction rather than re-injecting the exception, as that can cause premature
12617	* trips to ring-3 before injection and involve TRPM which currently has no way of
12618	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12619	* the problem).
12620	*/
12621	IEMXCPTRAISE enmRaise;
12622	IEMXCPTRAISEINFO fRaiseInfo;
12623	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12624	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12625	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12626	{
12627	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12628	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12629	}
12630	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12631	{
12632	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12633	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12634	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12635	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12636	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12637	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12638
12639	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12640
12641	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12642	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12643	{
12644	pVmxTransient->fVectoringPF = true;
12645	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12646	}
12647	}
12648	else
12649	{
12650	/*
12651	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12652	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12653	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12654	*/
12655	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12656	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12657	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12658	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12659	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12660	}
12661
12662	/*
12663	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12664	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12665	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12666	* subsequent VM-entry would fail.
12667	*
12668	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12669	*/
12670	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12671	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12672	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12673	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12674	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12675	{
12676	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12677	}
12678
12679	switch (enmRaise)
12680	{
12681	case IEMXCPTRAISE_CURRENT_XCPT:
12682	{
12683	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12684	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12685	Assert(rcStrict == VINF_SUCCESS);
12686	break;
12687	}
12688
12689	case IEMXCPTRAISE_PREV_EVENT:
12690	{
12691	uint32_t u32ErrCode;
12692	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12693	{
12694	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12695	AssertRCReturn(rc2, rc2);
12696	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12697	}
12698	else
12699	u32ErrCode = 0;
12700
12701	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12702	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12703	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12704	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12705
12706	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12707	pVCpu->hm.s.Event.u32ErrCode));
12708	Assert(rcStrict == VINF_SUCCESS);
12709	break;
12710	}
12711
12712	case IEMXCPTRAISE_REEXEC_INSTR:
12713	Assert(rcStrict == VINF_SUCCESS);
12714	break;
12715
12716	case IEMXCPTRAISE_DOUBLE_FAULT:
12717	{
12718	/*
12719	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12720	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12721	*/
12722	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12723	{
12724	pVmxTransient->fVectoringDoublePF = true;
12725	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12726	pVCpu->cpum.GstCtx.cr2));
12727	rcStrict = VINF_SUCCESS;
12728	}
12729	else
12730	{
12731	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12732	hmR0VmxSetPendingXcptDF(pVCpu);
12733	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12734	uIdtVector, uExitVector));
12735	rcStrict = VINF_HM_DOUBLE_FAULT;
12736	}
12737	break;
12738	}
12739
12740	case IEMXCPTRAISE_TRIPLE_FAULT:
12741	{
12742	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12743	rcStrict = VINF_EM_RESET;
12744	break;
12745	}
12746
12747	case IEMXCPTRAISE_CPU_HANG:
12748	{
12749	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12750	rcStrict = VERR_EM_GUEST_CPU_HANG;
12751	break;
12752	}
12753
12754	default:
12755	{
12756	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12757	rcStrict = VERR_VMX_IPE_2;
12758	break;
12759	}
12760	}
12761	}
12762	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12763	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12764	&& uExitVector != X86_XCPT_DF
12765	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12766	{
12767	/*
12768	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12769	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12770	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12771	*/
12772	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12773	{
12774	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12775	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12776	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12777	}
12778	}
12779
12780	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12781	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12782	return rcStrict;
12783	}
12784
12785
12786	/** @name VM-exit handlers.
12787	* @{
12788	*/
12789	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12790	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12791	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12792
12793	/**
12794	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12795	*/
12796	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12797	{
12798	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12799	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12800	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12801	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12802	return VINF_SUCCESS;
12803	return VINF_EM_RAW_INTERRUPT;
12804	}
12805
12806
12807	/**
12808	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12809	*/
12810	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12811	{
12812	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12813	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12814
12815	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12816	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12817	AssertRCReturn(rc, rc);
12818
12819	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12820	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12821	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12822	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12823
12824	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12825	{
12826	/*
12827	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12828	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12829	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12830	*
12831	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12832	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12833	*/
12834	VMXDispatchHostNmi();
12835	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12836	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12837	return VINF_SUCCESS;
12838	}
12839
12840	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12841	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12842	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12843	{ /* likely */ }
12844	else
12845	{
12846	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12847	rcStrictRc1 = VINF_SUCCESS;
12848	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12849	return rcStrictRc1;
12850	}
12851
12852	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
12853	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12854	switch (uIntType)
12855	{
12856	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12857	Assert(uVector == X86_XCPT_DB);
12858	RT_FALL_THRU();
12859	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12860	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12861	RT_FALL_THRU();
12862	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12863	{
12864	/*
12865	* If there's any exception caused as a result of event injection, the resulting
12866	* secondary/final execption will be pending, we shall continue guest execution
12867	* after injecting the event. The page-fault case is complicated and we manually
12868	* handle any currently pending event in hmR0VmxExitXcptPF.
12869	*/
12870	if (!pVCpu->hm.s.Event.fPending)
12871	{ /* likely */ }
12872	else if (uVector != X86_XCPT_PF)
12873	{
12874	rc = VINF_SUCCESS;
12875	break;
12876	}
12877
12878	switch (uVector)
12879	{
12880	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12881	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12882	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12883	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12884	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12885	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12886
12887	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12888	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12889	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12890	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12891	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12892	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12893	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12894	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12895	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12896	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12897	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12898	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12899	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12900	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12901	default:
12902	{
12903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12904	if (pVmcsInfo->RealMode.fRealOnV86Active)
12905	{
12906	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12907	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12908	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12909
12910	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12911	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12912	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12913	AssertRCReturn(rc, rc);
12914	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12915	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12916	0 /* GCPtrFaultAddress */);
12917	}
12918	else
12919	{
12920	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12921	pVCpu->hm.s.u32HMError = uVector;
12922	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12923	}
12924	break;
12925	}
12926	}
12927	break;
12928	}
12929
12930	default:
12931	{
12932	pVCpu->hm.s.u32HMError = uExitIntInfo;
12933	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12934	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12935	break;
12936	}
12937	}
12938	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12939	return rc;
12940	}
12941
12942
12943	/**
12944	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12945	*/
12946	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12947	{
12948	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12949
12950	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12951	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12952	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12953	AssertRCReturn(rc, rc);
12954
12955	/* Evaluate and deliver pending events and resume guest execution. */
12956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12957	return VINF_SUCCESS;
12958	}
12959
12960
12961	/**
12962	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12963	*/
12964	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12965	{
12966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12967
12968	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12969	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12970	{
12971	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12972	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12973	}
12974
12975	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12976
12977	/*
12978	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12979	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12980	*/
12981	uint32_t fIntrState;
12982	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12983	AssertRCReturn(rc, rc);
12984	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12985	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12986	{
12987	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12988	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12989
12990	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12991	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12992	AssertRCReturn(rc, rc);
12993	}
12994
12995	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12996	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12997	AssertRCReturn(rc, rc);
12998
12999	/* Evaluate and deliver pending events and resume guest execution. */
13000	return VINF_SUCCESS;
13001	}
13002
13003
13004	/**
13005	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13006	*/
13007	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13008	{
13009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13010	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13011	}
13012
13013
13014	/**
13015	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13016	*/
13017	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13018	{
13019	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13020	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13021	}
13022
13023
13024	/**
13025	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13026	*/
13027	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13028	{
13029	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13030
13031	/*
13032	* Get the state we need and update the exit history entry.
13033	*/
13034	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13035	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13036	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13037	AssertRCReturn(rc, rc);
13038
13039	VBOXSTRICTRC rcStrict;
13040	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13041	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13042	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13043	if (!pExitRec)
13044	{
13045	/*
13046	* Regular CPUID instruction execution.
13047	*/
13048	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13049	if (rcStrict == VINF_SUCCESS)
13050	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13051	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13052	{
13053	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13054	rcStrict = VINF_SUCCESS;
13055	}
13056	}
13057	else
13058	{
13059	/*
13060	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13061	*/
13062	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13063	AssertRCReturn(rc2, rc2);
13064
13065	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13066	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13067
13068	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13069	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13070
13071	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13072	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13073	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13074	}
13075	return rcStrict;
13076	}
13077
13078
13079	/**
13080	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13081	*/
13082	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13083	{
13084	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13085
13086	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13087	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13088	AssertRCReturn(rc, rc);
13089
13090	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13091	return VINF_EM_RAW_EMULATE_INSTR;
13092
13093	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13094	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13095	}
13096
13097
13098	/**
13099	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13100	*/
13101	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13102	{
13103	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13104
13105	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13106	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13107	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13108	AssertRCReturn(rc, rc);
13109
13110	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13111	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13112	{
13113	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13114	we must reset offsetting on VM-entry. See @bugref{6634}. */
13115	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13116	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13118	}
13119	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13120	{
13121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13122	rcStrict = VINF_SUCCESS;
13123	}
13124	return rcStrict;
13125	}
13126
13127
13128	/**
13129	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13130	*/
13131	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13132	{
13133	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13134
13135	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13136	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13137	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13138	AssertRCReturn(rc, rc);
13139
13140	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13141	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13142	{
13143	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13144	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13145	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13146	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13147	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13148	}
13149	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13150	{
13151	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13152	rcStrict = VINF_SUCCESS;
13153	}
13154	return rcStrict;
13155	}
13156
13157
13158	/**
13159	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13160	*/
13161	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13162	{
13163	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13164
13165	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13166	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13167	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13168	AssertRCReturn(rc, rc);
13169
13170	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13171	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13172	if (RT_LIKELY(rc == VINF_SUCCESS))
13173	{
13174	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13175	Assert(pVmxTransient->cbInstr == 2);
13176	}
13177	else
13178	{
13179	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13180	rc = VERR_EM_INTERPRETER;
13181	}
13182	return rc;
13183	}
13184
13185
13186	/**
13187	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13188	*/
13189	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13190	{
13191	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13192
13193	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13194	if (EMAreHypercallInstructionsEnabled(pVCpu))
13195	{
13196	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13197	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13198	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13199	AssertRCReturn(rc, rc);
13200
13201	/* Perform the hypercall. */
13202	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13203	if (rcStrict == VINF_SUCCESS)
13204	{
13205	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13206	AssertRCReturn(rc, rc);
13207	}
13208	else
13209	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13210	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13211	\|\| RT_FAILURE(rcStrict));
13212
13213	/* If the hypercall changes anything other than guest's general-purpose registers,
13214	we would need to reload the guest changed bits here before VM-entry. */
13215	}
13216	else
13217	Log4Func(("Hypercalls not enabled\n"));
13218
13219	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13220	if (RT_FAILURE(rcStrict))
13221	{
13222	hmR0VmxSetPendingXcptUD(pVCpu);
13223	rcStrict = VINF_SUCCESS;
13224	}
13225
13226	return rcStrict;
13227	}
13228
13229
13230	/**
13231	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13232	*/
13233	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13234	{
13235	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13236	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13237
13238	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13239	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13240	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13241	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13242	AssertRCReturn(rc, rc);
13243
13244	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13245
13246	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13247	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13248	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13249	{
13250	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13251	rcStrict = VINF_SUCCESS;
13252	}
13253	else
13254	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13255	VBOXSTRICTRC_VAL(rcStrict)));
13256	return rcStrict;
13257	}
13258
13259
13260	/**
13261	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13262	*/
13263	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13264	{
13265	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13266
13267	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13268	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13269	AssertRCReturn(rc, rc);
13270
13271	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13272	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13273	if (RT_LIKELY(rc == VINF_SUCCESS))
13274	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13275	else
13276	{
13277	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13278	rc = VERR_EM_INTERPRETER;
13279	}
13280	return rc;
13281	}
13282
13283
13284	/**
13285	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13286	*/
13287	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13288	{
13289	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13290
13291	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13292	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13293	AssertRCReturn(rc, rc);
13294
13295	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13296	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13297	rc = VBOXSTRICTRC_VAL(rc2);
13298	if (RT_LIKELY( rc == VINF_SUCCESS
13299	\|\| rc == VINF_EM_HALT))
13300	{
13301	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13302	AssertRCReturn(rc3, rc3);
13303
13304	if ( rc == VINF_EM_HALT
13305	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13306	rc = VINF_SUCCESS;
13307	}
13308	else
13309	{
13310	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13311	rc = VERR_EM_INTERPRETER;
13312	}
13313	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13314	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13315	return rc;
13316	}
13317
13318
13319	/**
13320	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13321	*/
13322	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13323	{
13324	/*
13325	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13326	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13327	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13328	* VMX root operation. If we get here, something funny is going on.
13329	*
13330	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13331	*/
13332	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13333	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13334	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13335	}
13336
13337
13338	/**
13339	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13340	*/
13341	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13342	{
13343	/*
13344	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13345	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13346	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13347	* an SMI. If we get here, something funny is going on.
13348	*
13349	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13350	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13351	*/
13352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13353	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13354	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13355	}
13356
13357
13358	/**
13359	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13360	*/
13361	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13362	{
13363	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13364	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13365	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13366	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13367	}
13368
13369
13370	/**
13371	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13372	*/
13373	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13374	{
13375	/*
13376	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13377	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13378	* See Intel spec. 25.3 "Other Causes of VM-exits".
13379	*/
13380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13381	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13382	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13383	}
13384
13385
13386	/**
13387	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13388	* VM-exit.
13389	*/
13390	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13391	{
13392	/*
13393	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13394	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13395	*
13396	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13397	* See Intel spec. "23.8 Restrictions on VMX operation".
13398	*/
13399	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13400	return VINF_SUCCESS;
13401	}
13402
13403
13404	/**
13405	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13406	* VM-exit.
13407	*/
13408	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13409	{
13410	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13411	return VINF_EM_RESET;
13412	}
13413
13414
13415	/**
13416	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13417	*/
13418	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13419	{
13420	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13421
13422	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13423	AssertRCReturn(rc, rc);
13424
13425	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13426	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13427	rc = VINF_SUCCESS;
13428	else
13429	rc = VINF_EM_HALT;
13430
13431	if (rc != VINF_SUCCESS)
13432	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13433	return rc;
13434	}
13435
13436
13437	/**
13438	* VM-exit handler for instructions that result in a \#UD exception delivered to
13439	* the guest.
13440	*/
13441	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13442	{
13443	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13444	hmR0VmxSetPendingXcptUD(pVCpu);
13445	return VINF_SUCCESS;
13446	}
13447
13448
13449	/**
13450	* VM-exit handler for expiry of the VMX-preemption timer.
13451	*/
13452	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13453	{
13454	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13455
13456	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13457	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13458
13459	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13460	PVM pVM = pVCpu->CTX_SUFF(pVM);
13461	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13463	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13464	}
13465
13466
13467	/**
13468	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13469	*/
13470	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13471	{
13472	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13473
13474	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13475	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13476	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13477	AssertRCReturn(rc, rc);
13478
13479	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13480	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13481	: HM_CHANGED_RAISED_XCPT_MASK);
13482
13483	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13484	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13485
13486	return rcStrict;
13487	}
13488
13489
13490	/**
13491	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13492	*/
13493	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13494	{
13495	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13496	/** @todo Use VM-exit instruction information. */
13497	return VERR_EM_INTERPRETER;
13498	}
13499
13500
13501	/**
13502	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13503	* Error VM-exit.
13504	*/
13505	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13506	{
13507	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13508	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13509	AssertRCReturn(rc, rc);
13510
13511	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13512	if (RT_FAILURE(rc))
13513	return rc;
13514
13515	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13516	NOREF(uInvalidReason);
13517
13518	#ifdef VBOX_STRICT
13519	uint32_t fIntrState;
13520	RTHCUINTREG uHCReg;
13521	uint64_t u64Val;
13522	uint32_t u32Val;
13523	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13524	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13525	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13526	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13527	AssertRCReturn(rc, rc);
13528
13529	Log4(("uInvalidReason %u\n", uInvalidReason));
13530	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13531	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13532	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13533	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13534
13535	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13536	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13537	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13538	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13539	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13540	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13541	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13542	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13543	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13544	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13545	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13546	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13547
13548	hmR0DumpRegs(pVCpu);
13549	#endif
13550
13551	return VERR_VMX_INVALID_GUEST_STATE;
13552	}
13553
13554
13555	/**
13556	* VM-exit handler for VM-entry failure due to an MSR-load
13557	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13558	*/
13559	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13560	{
13561	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13562	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13563	}
13564
13565
13566	/**
13567	* VM-exit handler for VM-entry failure due to a machine-check event
13568	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13569	*/
13570	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13571	{
13572	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13573	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13574	}
13575
13576
13577	/**
13578	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13579	* theory.
13580	*/
13581	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13582	{
13583	RT_NOREF2(pVCpu, pVmxTransient);
13584	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13585	return VERR_VMX_UNDEFINED_EXIT_CODE;
13586	}
13587
13588
13589	/**
13590	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13591	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13592	* Conditional VM-exit.
13593	*/
13594	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13595	{
13596	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13597
13598	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13600	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13601	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13602	return VERR_EM_INTERPRETER;
13603	AssertMsgFailed(("Unexpected XDTR access\n"));
13604	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13605	}
13606
13607
13608	/**
13609	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13610	*/
13611	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13612	{
13613	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13614
13615	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13616	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13617	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13618	return VERR_EM_INTERPRETER;
13619	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13620	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13621	}
13622
13623
13624	/**
13625	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13626	*/
13627	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13628	{
13629	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13630
13631	/** @todo Optimize this: We currently drag in in the whole MSR state
13632	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13633	* MSRs required. That would require changes to IEM and possibly CPUM too.
13634	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13635	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13636	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13637	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13638	switch (idMsr)
13639	{
13640	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13641	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13642	}
13643
13644	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13645	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13646	AssertRCReturn(rc, rc);
13647
13648	Log4Func(("ecx=%#RX32\n", idMsr));
13649
13650	#ifdef VBOX_STRICT
13651	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13652	{
13653	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13654	&& idMsr != MSR_K6_EFER)
13655	{
13656	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13657	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13658	}
13659	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13660	{
13661	Assert(pVmcsInfo->pvMsrBitmap);
13662	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13663	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13664	{
13665	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13666	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13667	}
13668	}
13669	}
13670	#endif
13671
13672	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13674	if (rcStrict == VINF_SUCCESS)
13675	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13676	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13677	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13678	{
13679	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13680	rcStrict = VINF_SUCCESS;
13681	}
13682	else
13683	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13684
13685	return rcStrict;
13686	}
13687
13688
13689	/**
13690	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13691	*/
13692	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13693	{
13694	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13695
13696	/** @todo Optimize this: We currently drag in in the whole MSR state
13697	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13698	* MSRs required. That would require changes to IEM and possibly CPUM too.
13699	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13700	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13701	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13702
13703	/*
13704	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13705	* Although we don't need to fetch the base as it will be overwritten shortly, while
13706	* loading guest-state we would also load the entire segment register including limit
13707	* and attributes and thus we need to load them here.
13708	*/
13709	switch (idMsr)
13710	{
13711	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13712	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13713	}
13714
13715	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13716	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13717	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13718	AssertRCReturn(rc, rc);
13719
13720	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13721
13722	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13724
13725	if (rcStrict == VINF_SUCCESS)
13726	{
13727	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13728
13729	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13730	if ( idMsr == MSR_IA32_APICBASE
13731	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13732	&& idMsr <= MSR_IA32_X2APIC_END))
13733	{
13734	/*
13735	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13736	* When full APIC register virtualization is implemented we'll have to make
13737	* sure APIC state is saved from the VMCS before IEM changes it.
13738	*/
13739	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13740	}
13741	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13742	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13743	else if (idMsr == MSR_K6_EFER)
13744	{
13745	/*
13746	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13747	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13748	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13749	*/
13750	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13751	}
13752
13753	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13754	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13755	{
13756	switch (idMsr)
13757	{
13758	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13759	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13760	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13761	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13762	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13763	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13764	default:
13765	{
13766	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13767	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13768	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13769	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13770	break;
13771	}
13772	}
13773	}
13774	#ifdef VBOX_STRICT
13775	else
13776	{
13777	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13778	switch (idMsr)
13779	{
13780	case MSR_IA32_SYSENTER_CS:
13781	case MSR_IA32_SYSENTER_EIP:
13782	case MSR_IA32_SYSENTER_ESP:
13783	case MSR_K8_FS_BASE:
13784	case MSR_K8_GS_BASE:
13785	{
13786	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
13787	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13788	}
13789
13790	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13791	default:
13792	{
13793	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13794	{
13795	/* EFER MSR writes are always intercepted. */
13796	if (idMsr != MSR_K6_EFER)
13797	{
13798	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13799	idMsr));
13800	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13801	}
13802	}
13803
13804	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13805	{
13806	Assert(pVmcsInfo->pvMsrBitmap);
13807	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13808	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13809	{
13810	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13811	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13812	}
13813	}
13814	break;
13815	}
13816	}
13817	}
13818	#endif /* VBOX_STRICT */
13819	}
13820	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13821	{
13822	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13823	rcStrict = VINF_SUCCESS;
13824	}
13825	else
13826	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13827
13828	return rcStrict;
13829	}
13830
13831
13832	/**
13833	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13834	*/
13835	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13836	{
13837	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13838	/** @todo The guest has likely hit a contended spinlock. We might want to
13839	* poke a schedule different guest VCPU. */
13840	return VINF_EM_RAW_INTERRUPT;
13841	}
13842
13843
13844	/**
13845	* VM-exit handler for when the TPR value is lowered below the specified
13846	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13847	*/
13848	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13849	{
13850	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13851	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13852
13853	/*
13854	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13855	* We'll re-evaluate pending interrupts and inject them before the next VM
13856	* entry so we can just continue execution here.
13857	*/
13858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13859	return VINF_SUCCESS;
13860	}
13861
13862
13863	/**
13864	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13865	* VM-exit.
13866	*
13867	* @retval VINF_SUCCESS when guest execution can continue.
13868	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13869	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13870	* interpreter.
13871	*/
13872	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13873	{
13874	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13875	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13876
13877	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13878	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13879	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13880	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13881	AssertRCReturn(rc, rc);
13882
13883	VBOXSTRICTRC rcStrict;
13884	PVM pVM = pVCpu->CTX_SUFF(pVM);
13885	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13886	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13887	switch (uAccessType)
13888	{
13889	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13890	{
13891	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13892	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13893	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13894	AssertMsg( rcStrict == VINF_SUCCESS
13895	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13896	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13897
13898	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13899	{
13900	case 0:
13901	{
13902	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13903	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13904	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13905	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13906
13907	/*
13908	* This is a kludge for handling switches back to real mode when we try to use
13909	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13910	* deal with special selector values, so we have to return to ring-3 and run
13911	* there till the selector values are V86 mode compatible.
13912	*
13913	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13914	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13915	* at the end of this function.
13916	*/
13917	if ( rc == VINF_SUCCESS
13918	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13919	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13920	&& (uOldCr0 & X86_CR0_PE)
13921	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13922	{
13923	/** @todo check selectors rather than returning all the time. */
13924	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13925	rcStrict = VINF_EM_RESCHEDULE_REM;
13926	}
13927	break;
13928	}
13929
13930	case 2:
13931	{
13932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13933	/* Nothing to do here, CR2 it's not part of the VMCS. */
13934	break;
13935	}
13936
13937	case 3:
13938	{
13939	Assert( !pVM->hm.s.fNestedPaging
13940	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13941	\|\| pVCpu->hm.s.fUsingDebugLoop);
13942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13943	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13944	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13945	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13946	break;
13947	}
13948
13949	case 4:
13950	{
13951	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13952	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13953	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13954	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13955	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13956	break;
13957	}
13958
13959	case 8:
13960	{
13961	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13962	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13963	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13964	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13965	break;
13966	}
13967	default:
13968	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13969	break;
13970	}
13971	break;
13972	}
13973
13974	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13975	{
13976	Assert( !pVM->hm.s.fNestedPaging
13977	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13978	\|\| pVCpu->hm.s.fUsingDebugLoop
13979	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13980	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13981	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13982	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13983
13984	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13985	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13986	AssertMsg( rcStrict == VINF_SUCCESS
13987	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13988	#ifdef VBOX_WITH_STATISTICS
13989	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13990	{
13991	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13992	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13993	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13994	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13995	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13996	}
13997	#endif
13998	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13999	VBOXSTRICTRC_VAL(rcStrict)));
14000	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14001	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14002	else
14003	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14004	break;
14005	}
14006
14007	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14008	{
14009	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14010	AssertMsg( rcStrict == VINF_SUCCESS
14011	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14012
14013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14014	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14015	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14016	break;
14017	}
14018
14019	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14020	{
14021	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14022	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14023	AssertRCReturn(rc, rc);
14024	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14025	pVmxTransient->uGuestLinearAddr);
14026	AssertMsg( rcStrict == VINF_SUCCESS
14027	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14028	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14029
14030	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14031	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14032	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14033	break;
14034	}
14035
14036	default:
14037	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14038	VERR_VMX_UNEXPECTED_EXCEPTION);
14039	}
14040
14041	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14042	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14043	if (rcStrict == VINF_IEM_RAISED_XCPT)
14044	{
14045	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14046	rcStrict = VINF_SUCCESS;
14047	}
14048
14049	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14050	NOREF(pVM);
14051	return rcStrict;
14052	}
14053
14054
14055	/**
14056	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14057	* VM-exit.
14058	*/
14059	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14060	{
14061	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14062	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14063
14064	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14065	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14066	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14067	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14068	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14069	\| CPUMCTX_EXTRN_EFER);
14070	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14071	AssertRCReturn(rc, rc);
14072
14073	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14074	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14075	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14076	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14077	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14078	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14079	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14080	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14081
14082	/*
14083	* Update exit history to see if this exit can be optimized.
14084	*/
14085	VBOXSTRICTRC rcStrict;
14086	PCEMEXITREC pExitRec = NULL;
14087	if ( !fGstStepping
14088	&& !fDbgStepping)
14089	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14090	!fIOString
14091	? !fIOWrite
14092	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14093	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14094	: !fIOWrite
14095	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14096	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14097	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14098	if (!pExitRec)
14099	{
14100	/* I/O operation lookup arrays. */
14101	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14102	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14103	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14104	uint32_t const cbInstr = pVmxTransient->cbInstr;
14105	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14106	PVM pVM = pVCpu->CTX_SUFF(pVM);
14107	if (fIOString)
14108	{
14109	/*
14110	* INS/OUTS - I/O String instruction.
14111	*
14112	* Use instruction-information if available, otherwise fall back on
14113	* interpreting the instruction.
14114	*/
14115	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14116	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14117	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14118	if (fInsOutsInfo)
14119	{
14120	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14121	AssertRCReturn(rc2, rc2);
14122	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14123	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14124	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14125	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14126	if (fIOWrite)
14127	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14128	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14129	else
14130	{
14131	/*
14132	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14133	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14134	* See Intel Instruction spec. for "INS".
14135	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14136	*/
14137	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14138	}
14139	}
14140	else
14141	rcStrict = IEMExecOne(pVCpu);
14142
14143	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14144	fUpdateRipAlready = true;
14145	}
14146	else
14147	{
14148	/*
14149	* IN/OUT - I/O instruction.
14150	*/
14151	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14152	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14153	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14154	if (fIOWrite)
14155	{
14156	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14158	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14159	&& !pCtx->eflags.Bits.u1TF)
14160	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14161	}
14162	else
14163	{
14164	uint32_t u32Result = 0;
14165	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14166	if (IOM_SUCCESS(rcStrict))
14167	{
14168	/* Save result of I/O IN instr. in AL/AX/EAX. */
14169	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14170	}
14171	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14172	&& !pCtx->eflags.Bits.u1TF)
14173	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14175	}
14176	}
14177
14178	if (IOM_SUCCESS(rcStrict))
14179	{
14180	if (!fUpdateRipAlready)
14181	{
14182	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14183	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14184	}
14185
14186	/*
14187	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14188	* while booting Fedora 17 64-bit guest.
14189	*
14190	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14191	*/
14192	if (fIOString)
14193	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14194
14195	/*
14196	* If any I/O breakpoints are armed, we need to check if one triggered
14197	* and take appropriate action.
14198	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14199	*/
14200	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14201	AssertRCReturn(rc, rc);
14202
14203	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14204	* execution engines about whether hyper BPs and such are pending. */
14205	uint32_t const uDr7 = pCtx->dr[7];
14206	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14207	&& X86_DR7_ANY_RW_IO(uDr7)
14208	&& (pCtx->cr4 & X86_CR4_DE))
14209	\|\| DBGFBpIsHwIoArmed(pVM)))
14210	{
14211	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14212
14213	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14214	VMMRZCallRing3Disable(pVCpu);
14215	HM_DISABLE_PREEMPT(pVCpu);
14216
14217	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14218
14219	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14220	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14221	{
14222	/* Raise #DB. */
14223	if (fIsGuestDbgActive)
14224	ASMSetDR6(pCtx->dr[6]);
14225	if (pCtx->dr[7] != uDr7)
14226	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14227
14228	hmR0VmxSetPendingXcptDB(pVCpu);
14229	}
14230	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14231	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14232	else if ( rcStrict2 != VINF_SUCCESS
14233	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14234	rcStrict = rcStrict2;
14235	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14236
14237	HM_RESTORE_PREEMPT();
14238	VMMRZCallRing3Enable(pVCpu);
14239	}
14240	}
14241
14242	#ifdef VBOX_STRICT
14243	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14244	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14245	Assert(!fIOWrite);
14246	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14247	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14248	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14249	Assert(fIOWrite);
14250	else
14251	{
14252	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14253	* statuses, that the VMM device and some others may return. See
14254	* IOM_SUCCESS() for guidance. */
14255	AssertMsg( RT_FAILURE(rcStrict)
14256	\|\| rcStrict == VINF_SUCCESS
14257	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14258	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14259	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14260	\|\| rcStrict == VINF_EM_RAW_TO_R3
14261	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14262	# endif
14263	}
14264	#endif
14265	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14266	}
14267	else
14268	{
14269	/*
14270	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14271	*/
14272	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14273	AssertRCReturn(rc2, rc2);
14274	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14275	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14276	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14277	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14278	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14279	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14280
14281	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14282	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14283
14284	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14285	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14286	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14287	}
14288	return rcStrict;
14289	}
14290
14291
14292	/**
14293	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14294	* VM-exit.
14295	*/
14296	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14297	{
14298	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14299
14300	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14301	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14302	AssertRCReturn(rc, rc);
14303	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14304	{
14305	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14306	AssertRCReturn(rc, rc);
14307	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14308	{
14309	uint32_t uErrCode;
14310	RTGCUINTPTR GCPtrFaultAddress;
14311	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14312	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14313	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14314	if (fErrorCodeValid)
14315	{
14316	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14317	AssertRCReturn(rc, rc);
14318	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14319	}
14320	else
14321	uErrCode = 0;
14322
14323	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14324	&& uVector == X86_XCPT_PF)
14325	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14326	else
14327	GCPtrFaultAddress = 0;
14328
14329	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14330	AssertRCReturn(rc, rc);
14331
14332	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14333	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14334
14335	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14337	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14338	}
14339	}
14340
14341	/* Fall back to the interpreter to emulate the task-switch. */
14342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14343	return VERR_EM_INTERPRETER;
14344	}
14345
14346
14347	/**
14348	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14349	*/
14350	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14351	{
14352	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14353
14354	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14355	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14356	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14357	AssertRCReturn(rc, rc);
14358	return VINF_EM_DBG_STEPPED;
14359	}
14360
14361
14362	/**
14363	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14364	*/
14365	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14366	{
14367	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14369
14370	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14371	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14372	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14373	{
14374	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14375	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14376	{
14377	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14378	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14379	}
14380	}
14381	else
14382	{
14383	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14384	rcStrict1 = VINF_SUCCESS;
14385	return rcStrict1;
14386	}
14387
14388	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14389	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14390	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14391	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14392	AssertRCReturn(rc, rc);
14393
14394	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14395	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14396	VBOXSTRICTRC rcStrict2;
14397	switch (uAccessType)
14398	{
14399	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14400	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14401	{
14402	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14403	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14404	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14405
14406	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14407	GCPhys &= PAGE_BASE_GC_MASK;
14408	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14409	PVM pVM = pVCpu->CTX_SUFF(pVM);
14410	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14411	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14412
14413	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14414	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14415	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14416	CPUMCTX2CORE(pCtx), GCPhys);
14417	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14418	if ( rcStrict2 == VINF_SUCCESS
14419	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14420	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14421	{
14422	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14423	\| HM_CHANGED_GUEST_APIC_TPR);
14424	rcStrict2 = VINF_SUCCESS;
14425	}
14426	break;
14427	}
14428
14429	default:
14430	Log4Func(("uAccessType=%#x\n", uAccessType));
14431	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14432	break;
14433	}
14434
14435	if (rcStrict2 != VINF_SUCCESS)
14436	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14437	return rcStrict2;
14438	}
14439
14440
14441	/**
14442	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14443	* VM-exit.
14444	*/
14445	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14446	{
14447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14448
14449	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14450	if (pVmxTransient->fWasGuestDebugStateActive)
14451	{
14452	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14453	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14454	}
14455
14456	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14457	if ( !pVCpu->hm.s.fSingleInstruction
14458	&& !pVmxTransient->fWasHyperDebugStateActive)
14459	{
14460	Assert(!DBGFIsStepping(pVCpu));
14461	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14462
14463	/* Don't intercept MOV DRx any more. */
14464	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14465	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14466	AssertRCReturn(rc, rc);
14467
14468	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14469	VMMRZCallRing3Disable(pVCpu);
14470	HM_DISABLE_PREEMPT(pVCpu);
14471
14472	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14473	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14474	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14475
14476	HM_RESTORE_PREEMPT();
14477	VMMRZCallRing3Enable(pVCpu);
14478
14479	#ifdef VBOX_WITH_STATISTICS
14480	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14481	AssertRCReturn(rc, rc);
14482	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14484	else
14485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14486	#endif
14487	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14488	return VINF_SUCCESS;
14489	}
14490
14491	/*
14492	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14493	* The EFER MSR is always up-to-date.
14494	* Update the segment registers and DR7 from the CPU.
14495	*/
14496	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14497	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14498	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14499	AssertRCReturn(rc, rc);
14500	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14501
14502	PVM pVM = pVCpu->CTX_SUFF(pVM);
14503	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14504	{
14505	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14506	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14507	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14508	if (RT_SUCCESS(rc))
14509	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14511	}
14512	else
14513	{
14514	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14515	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14516	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14518	}
14519
14520	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14521	if (RT_SUCCESS(rc))
14522	{
14523	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14524	AssertRCReturn(rc2, rc2);
14525	return VINF_SUCCESS;
14526	}
14527	return rc;
14528	}
14529
14530
14531	/**
14532	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14533	* Conditional VM-exit.
14534	*/
14535	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14536	{
14537	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14538	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14539
14540	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14541	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14542	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14543	{
14544	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14545	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14546	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14547	{
14548	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14549	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14550	}
14551	}
14552	else
14553	{
14554	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14555	rcStrict1 = VINF_SUCCESS;
14556	return rcStrict1;
14557	}
14558
14559	/*
14560	* Get sufficent state and update the exit history entry.
14561	*/
14562	RTGCPHYS GCPhys;
14563	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14564	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14565	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14566	AssertRCReturn(rc, rc);
14567
14568	VBOXSTRICTRC rcStrict;
14569	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14570	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14571	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14572	if (!pExitRec)
14573	{
14574	/*
14575	* If we succeed, resume guest execution.
14576	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14577	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14578	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14579	* weird case. See @bugref{6043}.
14580	*/
14581	PVM pVM = pVCpu->CTX_SUFF(pVM);
14582	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14583	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14584	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14585	if ( rcStrict == VINF_SUCCESS
14586	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14587	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14588	{
14589	/* Successfully handled MMIO operation. */
14590	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14591	\| HM_CHANGED_GUEST_APIC_TPR);
14592	rcStrict = VINF_SUCCESS;
14593	}
14594	}
14595	else
14596	{
14597	/*
14598	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14599	*/
14600	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14601	AssertRCReturn(rc2, rc2);
14602
14603	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14604	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14605
14606	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14607	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14608
14609	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14610	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14611	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14612	}
14613	return VBOXSTRICTRC_TODO(rcStrict);
14614	}
14615
14616
14617	/**
14618	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14619	* VM-exit.
14620	*/
14621	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14622	{
14623	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14624	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14625
14626	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14627	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14628	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14629	{
14630	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14631	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14632	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14633	}
14634	else
14635	{
14636	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14637	rcStrict1 = VINF_SUCCESS;
14638	return rcStrict1;
14639	}
14640
14641	RTGCPHYS GCPhys;
14642	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14643	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14644	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14645	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14646	AssertRCReturn(rc, rc);
14647
14648	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14649	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14650
14651	RTGCUINT uErrorCode = 0;
14652	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14653	uErrorCode \|= X86_TRAP_PF_ID;
14654	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14655	uErrorCode \|= X86_TRAP_PF_RW;
14656	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14657	uErrorCode \|= X86_TRAP_PF_P;
14658
14659	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14660
14661
14662	/* Handle the pagefault trap for the nested shadow table. */
14663	PVM pVM = pVCpu->CTX_SUFF(pVM);
14664	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14665
14666	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14667	pCtx->cs.Sel, pCtx->rip));
14668
14669	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14670	TRPMResetTrap(pVCpu);
14671
14672	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14673	if ( rcStrict2 == VINF_SUCCESS
14674	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14675	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14676	{
14677	/* Successfully synced our nested page tables. */
14678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14679	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14680	return VINF_SUCCESS;
14681	}
14682
14683	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14684	return rcStrict2;
14685	}
14686
14687	/** @} */
14688
14689	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14690	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14691	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14692
14693	/**
14694	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14695	*/
14696	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14697	{
14698	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14700
14701	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14702	AssertRCReturn(rc, rc);
14703
14704	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14705	{
14706	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14707	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14708
14709	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14710	* provides VM-exit instruction length. If this causes problem later,
14711	* disassemble the instruction like it's done on AMD-V. */
14712	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14713	AssertRCReturn(rc2, rc2);
14714	return rc;
14715	}
14716
14717	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14718	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14719	return rc;
14720	}
14721
14722
14723	/**
14724	* VM-exit exception handler for \#BP (Breakpoint exception).
14725	*/
14726	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14727	{
14728	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14730
14731	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14732	AssertRCReturn(rc, rc);
14733
14734	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14735	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14736	if (rc == VINF_EM_RAW_GUEST_TRAP)
14737	{
14738	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14739	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14740	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14741	AssertRCReturn(rc, rc);
14742
14743	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14744	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14745	}
14746
14747	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14748	return rc;
14749	}
14750
14751
14752	/**
14753	* VM-exit exception handler for \#AC (alignment check exception).
14754	*/
14755	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14756	{
14757	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14758
14759	/*
14760	* Re-inject it. We'll detect any nesting before getting here.
14761	*/
14762	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14763	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14764	AssertRCReturn(rc, rc);
14765	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14766
14767	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14768	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14769	return VINF_SUCCESS;
14770	}
14771
14772
14773	/**
14774	* VM-exit exception handler for \#DB (Debug exception).
14775	*/
14776	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14777	{
14778	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14780
14781	/*
14782	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14783	* for processing.
14784	*/
14785	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14786
14787	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14788	uint64_t const uDR6 = X86_DR6_INIT_VAL
14789	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14790	\| X86_DR6_BD \| X86_DR6_BS));
14791
14792	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14793	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14794	Log6Func(("rc=%Rrc\n", rc));
14795	if (rc == VINF_EM_RAW_GUEST_TRAP)
14796	{
14797	/*
14798	* The exception was for the guest. Update DR6, DR7.GD and
14799	* IA32_DEBUGCTL.LBR before forwarding it.
14800	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14801	*/
14802	VMMRZCallRing3Disable(pVCpu);
14803	HM_DISABLE_PREEMPT(pVCpu);
14804
14805	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14806	pCtx->dr[6] \|= uDR6;
14807	if (CPUMIsGuestDebugStateActive(pVCpu))
14808	ASMSetDR6(pCtx->dr[6]);
14809
14810	HM_RESTORE_PREEMPT();
14811	VMMRZCallRing3Enable(pVCpu);
14812
14813	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14814	AssertRCReturn(rc, rc);
14815
14816	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14817	pCtx->dr[7] &= ~X86_DR7_GD;
14818
14819	/* Paranoia. */
14820	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14821	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14822
14823	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14824	AssertRCReturn(rc, rc);
14825
14826	/*
14827	* Raise #DB in the guest.
14828	*
14829	* It is important to reflect exactly what the VM-exit gave us (preserving the
14830	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14831	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14832	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14833	*
14834	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14835	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14836	*/
14837	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14838	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14839	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14840	AssertRCReturn(rc, rc);
14841	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14842	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14843	return VINF_SUCCESS;
14844	}
14845
14846	/*
14847	* Not a guest trap, must be a hypervisor related debug event then.
14848	* Update DR6 in case someone is interested in it.
14849	*/
14850	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14851	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14852	CPUMSetHyperDR6(pVCpu, uDR6);
14853
14854	return rc;
14855	}
14856
14857
14858	/**
14859	* Hacks its way around the lovely mesa driver's backdoor accesses.
14860	*
14861	* @sa hmR0SvmHandleMesaDrvGp
14862	*/
14863	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14864	{
14865	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14866	RT_NOREF(pCtx);
14867
14868	/* For now we'll just skip the instruction. */
14869	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14870	}
14871
14872
14873	/**
14874	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14875	* backdoor logging w/o checking what it is running inside.
14876	*
14877	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14878	* backdoor port and magic numbers loaded in registers.
14879	*
14880	* @returns true if it is, false if it isn't.
14881	* @sa hmR0SvmIsMesaDrvGp
14882	*/
14883	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14884	{
14885	/* 0xed: IN eAX,dx */
14886	uint8_t abInstr[1];
14887	if (pVmxTransient->cbInstr != sizeof(abInstr))
14888	return false;
14889
14890	/* Check that it is #GP(0). */
14891	if (pVmxTransient->uExitIntErrorCode != 0)
14892	return false;
14893
14894	/* Check magic and port. */
14895	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14896	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14897	if (pCtx->rax != UINT32_C(0x564d5868))
14898	return false;
14899	if (pCtx->dx != UINT32_C(0x5658))
14900	return false;
14901
14902	/* Flat ring-3 CS. */
14903	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14904	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14905	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14906	if (pCtx->cs.Attr.n.u2Dpl != 3)
14907	return false;
14908	if (pCtx->cs.u64Base != 0)
14909	return false;
14910
14911	/* Check opcode. */
14912	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14913	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14914	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14915	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14916	if (RT_FAILURE(rc))
14917	return false;
14918	if (abInstr[0] != 0xed)
14919	return false;
14920
14921	return true;
14922	}
14923
14924
14925	/**
14926	* VM-exit exception handler for \#GP (General-protection exception).
14927	*
14928	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14929	*/
14930	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14931	{
14932	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14934
14935	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14936	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14937	if (pVmcsInfo->RealMode.fRealOnV86Active)
14938	{ /* likely */ }
14939	else
14940	{
14941	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14942	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14943	#endif
14944	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14945	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14946	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14947	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14948	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14949	AssertRCReturn(rc, rc);
14950	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14951	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14952
14953	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14954	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14955	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14956	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14957	else
14958	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14959	return rc;
14960	}
14961
14962	Assert(CPUMIsGuestInRealModeEx(pCtx));
14963	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14964
14965	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14966	AssertRCReturn(rc, rc);
14967
14968	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14969	if (rcStrict == VINF_SUCCESS)
14970	{
14971	if (!CPUMIsGuestInRealModeEx(pCtx))
14972	{
14973	/*
14974	* The guest is no longer in real-mode, check if we can continue executing the
14975	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14976	*/
14977	pVmcsInfo->RealMode.fRealOnV86Active = false;
14978	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14979	{
14980	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14982	}
14983	else
14984	{
14985	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14986	rcStrict = VINF_EM_RESCHEDULE;
14987	}
14988	}
14989	else
14990	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14991	}
14992	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14993	{
14994	rcStrict = VINF_SUCCESS;
14995	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14996	}
14997	return VBOXSTRICTRC_VAL(rcStrict);
14998	}
14999
15000
15001	/**
15002	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15003	* the exception reported in the VMX transient structure back into the VM.
15004	*
15005	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15006	* up-to-date.
15007	*/
15008	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15009	{
15010	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15011	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15012	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15013	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15014	("uVector=%#x u32XcptBitmap=%#X32\n",
15015	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15016	NOREF(pVmcsInfo);
15017	#endif
15018
15019	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15020	hmR0VmxCheckExitDueToEventDelivery(). */
15021	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15022	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15023	AssertRCReturn(rc, rc);
15024	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15025
15026	#ifdef DEBUG_ramshankar
15027	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15028	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15029	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15030	#endif
15031
15032	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15033	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15034	return VINF_SUCCESS;
15035	}
15036
15037
15038	/**
15039	* VM-exit exception handler for \#PF (Page-fault exception).
15040	*/
15041	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15042	{
15043	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15044	PVM pVM = pVCpu->CTX_SUFF(pVM);
15045	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15046	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15047	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15048	AssertRCReturn(rc, rc);
15049
15050	if (!pVM->hm.s.fNestedPaging)
15051	{ /* likely */ }
15052	else
15053	{
15054	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15055	Assert(pVCpu->hm.s.fUsingDebugLoop);
15056	#endif
15057	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15058	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15059	{
15060	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15061	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15062	}
15063	else
15064	{
15065	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15066	hmR0VmxSetPendingXcptDF(pVCpu);
15067	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15068	}
15069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15070	return rc;
15071	}
15072
15073	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15074	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15075	if (pVmxTransient->fVectoringPF)
15076	{
15077	Assert(pVCpu->hm.s.Event.fPending);
15078	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15079	}
15080
15081	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15082	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15083	AssertRCReturn(rc, rc);
15084
15085	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15086	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15087
15088	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15089	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15090
15091	Log4Func(("#PF: rc=%Rrc\n", rc));
15092	if (rc == VINF_SUCCESS)
15093	{
15094	/*
15095	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15096	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15097	*/
15098	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15099	TRPMResetTrap(pVCpu);
15100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15101	return rc;
15102	}
15103
15104	if (rc == VINF_EM_RAW_GUEST_TRAP)
15105	{
15106	if (!pVmxTransient->fVectoringDoublePF)
15107	{
15108	/* It's a guest page fault and needs to be reflected to the guest. */
15109	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15110	TRPMResetTrap(pVCpu);
15111	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15112	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15113	uGstErrorCode, pVmxTransient->uExitQual);
15114	}
15115	else
15116	{
15117	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15118	TRPMResetTrap(pVCpu);
15119	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15120	hmR0VmxSetPendingXcptDF(pVCpu);
15121	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15122	}
15123
15124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15125	return VINF_SUCCESS;
15126	}
15127
15128	TRPMResetTrap(pVCpu);
15129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15130	return rc;
15131	}
15132
15133	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15134	/** @name VMX instruction handlers.
15135	* @{
15136	*/
15137	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15138	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15139	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15140
15141	/**
15142	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15143	*/
15144	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15145	{
15146	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15147
15148	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15149	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15150	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15151	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15152	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15153	AssertRCReturn(rc, rc);
15154
15155	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15156
15157	VMXVEXITINFO ExitInfo;
15158	RT_ZERO(ExitInfo);
15159	ExitInfo.uReason = pVmxTransient->uExitReason;
15160	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15161	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15162	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15163	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15164
15165	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15166	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15167	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15168	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15169	{
15170	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15171	rcStrict = VINF_SUCCESS;
15172	}
15173	return rcStrict;
15174	}
15175
15176
15177	/**
15178	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15179	*/
15180	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15181	{
15182	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15183
15184	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15185	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15186	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15187	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15188	AssertRCReturn(rc, rc);
15189
15190	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15191
15192	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15193	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15194	{
15195	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15196	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15197	}
15198	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15199	return rcStrict;
15200	}
15201
15202
15203	/**
15204	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15205	*/
15206	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15207	{
15208	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15209
15210	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15211	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15212	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15213	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15214	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15215	AssertRCReturn(rc, rc);
15216
15217	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15218
15219	VMXVEXITINFO ExitInfo;
15220	RT_ZERO(ExitInfo);
15221	ExitInfo.uReason = pVmxTransient->uExitReason;
15222	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15223	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15224	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15225	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15226
15227	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15228	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15229	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15230	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15231	{
15232	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15233	rcStrict = VINF_SUCCESS;
15234	}
15235	return rcStrict;
15236	}
15237
15238
15239	/**
15240	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15241	*/
15242	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15243	{
15244	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15245
15246	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15247	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15248	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15249	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15250	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15251	AssertRCReturn(rc, rc);
15252
15253	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15254
15255	VMXVEXITINFO ExitInfo;
15256	RT_ZERO(ExitInfo);
15257	ExitInfo.uReason = pVmxTransient->uExitReason;
15258	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15259	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15260	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15261	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15262
15263	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15264	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15265	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15266	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15267	{
15268	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15269	rcStrict = VINF_SUCCESS;
15270	}
15271	return rcStrict;
15272	}
15273
15274
15275	/**
15276	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15277	*/
15278	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15279	{
15280	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15281
15282	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15283	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15284	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15285	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15286	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15287	AssertRCReturn(rc, rc);
15288
15289	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15290
15291	VMXVEXITINFO ExitInfo;
15292	RT_ZERO(ExitInfo);
15293	ExitInfo.uReason = pVmxTransient->uExitReason;
15294	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15295	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15296	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15297	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15298	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15299
15300	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15301	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15302	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15303	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15304	{
15305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15306	rcStrict = VINF_SUCCESS;
15307	}
15308	return rcStrict;
15309	}
15310
15311
15312	/**
15313	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15314	*/
15315	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15316	{
15317	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15318
15319	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15320	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15321	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15322	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15323	AssertRCReturn(rc, rc);
15324
15325	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15326
15327	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15328	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15329	{
15330	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15331	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15332	}
15333	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15334	return rcStrict;
15335	}
15336
15337
15338	/**
15339	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15340	*/
15341	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15342	{
15343	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15344
15345	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15346	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15347	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15348	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15349	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15350	AssertRCReturn(rc, rc);
15351
15352	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15353
15354	VMXVEXITINFO ExitInfo;
15355	RT_ZERO(ExitInfo);
15356	ExitInfo.uReason = pVmxTransient->uExitReason;
15357	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15358	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15359	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15360	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15361	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15362
15363	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15364	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15365	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15366	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15367	{
15368	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15369	rcStrict = VINF_SUCCESS;
15370	}
15371	return rcStrict;
15372	}
15373
15374
15375	/**
15376	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15377	*/
15378	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15379	{
15380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15381
15382	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15383	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15384	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15385	AssertRCReturn(rc, rc);
15386
15387	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15388
15389	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15390	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15391	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15392	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15393	{
15394	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15395	rcStrict = VINF_SUCCESS;
15396	}
15397	return rcStrict;
15398	}
15399
15400
15401	/**
15402	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15403	*/
15404	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15405	{
15406	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15407
15408	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15409	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15410	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15411	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15412	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15413	AssertRCReturn(rc, rc);
15414
15415	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15416
15417	VMXVEXITINFO ExitInfo;
15418	RT_ZERO(ExitInfo);
15419	ExitInfo.uReason = pVmxTransient->uExitReason;
15420	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15421	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15422	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15423	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15424
15425	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15426	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15427	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15428	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15429	{
15430	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15431	rcStrict = VINF_SUCCESS;
15432	}
15433	return rcStrict;
15434	}
15435
15436	/** @} */
15437	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15438

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78349

Download in other formats: