HMVMXR0.cpp@ 78680

Last change on this file since 78680 was 78679, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Fix PAUSE-loop exiting enabling.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 659.4 KB

Line
1	/* $Id: HMVMXR0.cpp 78679 2019-05-23 04:32:11Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
366	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369	static FNVMXEXITHANDLER hmR0VmxExitHlt;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
371	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
376	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
377	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
378	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
379	static FNVMXEXITHANDLER hmR0VmxExitVmread;
380	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
381	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
382	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
383	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
384	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
385	#endif
386	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
387	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
388	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
389	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
390	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
391	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
392	static FNVMXEXITHANDLER hmR0VmxExitMwait;
393	static FNVMXEXITHANDLER hmR0VmxExitMtf;
394	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395	static FNVMXEXITHANDLER hmR0VmxExitPause;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
404	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
410	/** @} */
411
412	/** @name Helpers for hardware exceptions VM-exit handlers.
413	* @{
414	*/
415	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
416	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
417	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
418	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
419	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
420	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
421	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	/** @} */
423
424
425	/*********************************************************************************************************************************
426	* Global Variables *
427	*********************************************************************************************************************************/
428	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
429	static const uint32_t g_aVmcsCacheSegBase[] =
430	{
431	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
432	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
433	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
434	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
435	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
436	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
437	};
438	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
439	#endif
440	static const uint32_t g_aVmcsSegBase[] =
441	{
442	VMX_VMCS_GUEST_ES_BASE,
443	VMX_VMCS_GUEST_CS_BASE,
444	VMX_VMCS_GUEST_SS_BASE,
445	VMX_VMCS_GUEST_DS_BASE,
446	VMX_VMCS_GUEST_FS_BASE,
447	VMX_VMCS_GUEST_GS_BASE
448	};
449	static const uint32_t g_aVmcsSegSel[] =
450	{
451	VMX_VMCS16_GUEST_ES_SEL,
452	VMX_VMCS16_GUEST_CS_SEL,
453	VMX_VMCS16_GUEST_SS_SEL,
454	VMX_VMCS16_GUEST_DS_SEL,
455	VMX_VMCS16_GUEST_FS_SEL,
456	VMX_VMCS16_GUEST_GS_SEL
457	};
458	static const uint32_t g_aVmcsSegLimit[] =
459	{
460	VMX_VMCS32_GUEST_ES_LIMIT,
461	VMX_VMCS32_GUEST_CS_LIMIT,
462	VMX_VMCS32_GUEST_SS_LIMIT,
463	VMX_VMCS32_GUEST_DS_LIMIT,
464	VMX_VMCS32_GUEST_FS_LIMIT,
465	VMX_VMCS32_GUEST_GS_LIMIT
466	};
467	static const uint32_t g_aVmcsSegAttr[] =
468	{
469	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
470	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
471	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
472	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
473	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
474	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
475	};
476	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
477	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
478	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
479	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
480
481	#ifdef HMVMX_USE_FUNCTION_TABLE
482	/**
483	* VMX_EXIT dispatch table.
484	*/
485	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
486	{
487	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
488	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
489	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
490	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
491	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
492	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
493	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
494	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
495	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
496	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
497	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
498	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
499	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
500	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
501	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
502	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
503	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
504	/* 17 VMX_EXIT_RSM */ hmR0VmxExitSetPendingXcptUD,
505	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
506	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
507	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
508	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
509	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
510	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
511	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
512	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
513	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
514	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
515	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
516	#else
517	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
518	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
519	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
520	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
521	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
522	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
523	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
524	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
525	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
526	#endif
527	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
528	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
529	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
530	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
531	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
532	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
533	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
534	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
535	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
536	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
537	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
538	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
539	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
540	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
541	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
542	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
543	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
544	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
545	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
546	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
547	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
548	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
549	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
550	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
551	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
552	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
553	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
554	#else
555	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
556	#endif
557	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
558	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
559	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
560	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
561	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
562	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
563	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitSetPendingXcptUD,
564	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitSetPendingXcptUD,
565	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
566	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
567	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
568	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
569	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
570	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitSetPendingXcptUD,
571	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitSetPendingXcptUD,
572	};
573	#endif /* HMVMX_USE_FUNCTION_TABLE */
574
575	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
576	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
577	{
578	/* 0 */ "(Not Used)",
579	/* 1 */ "VMCALL executed in VMX root operation.",
580	/* 2 */ "VMCLEAR with invalid physical address.",
581	/* 3 */ "VMCLEAR with VMXON pointer.",
582	/* 4 */ "VMLAUNCH with non-clear VMCS.",
583	/* 5 */ "VMRESUME with non-launched VMCS.",
584	/* 6 */ "VMRESUME after VMXOFF",
585	/* 7 */ "VM-entry with invalid control fields.",
586	/* 8 */ "VM-entry with invalid host state fields.",
587	/* 9 */ "VMPTRLD with invalid physical address.",
588	/* 10 */ "VMPTRLD with VMXON pointer.",
589	/* 11 */ "VMPTRLD with incorrect revision identifier.",
590	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
591	/* 13 */ "VMWRITE to read-only VMCS component.",
592	/* 14 */ "(Not Used)",
593	/* 15 */ "VMXON executed in VMX root operation.",
594	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
595	/* 17 */ "VM-entry with non-launched executing VMCS.",
596	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
597	/* 19 */ "VMCALL with non-clear VMCS.",
598	/* 20 */ "VMCALL with invalid VM-exit control fields.",
599	/* 21 */ "(Not Used)",
600	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
601	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
602	/* 24 */ "VMCALL with invalid SMM-monitor features.",
603	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
604	/* 26 */ "VM-entry with events blocked by MOV SS.",
605	/* 27 */ "(Not Used)",
606	/* 28 */ "Invalid operand to INVEPT/INVVPID."
607	};
608	#endif /* VBOX_STRICT */
609
610
611	/**
612	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
613	*
614	* Any bit set in this mask is owned by the host/hypervisor and would cause a
615	* VM-exit when modified by the guest.
616	*
617	* @returns The static CR0 guest/host mask.
618	* @param pVCpu The cross context virtual CPU structure.
619	*/
620	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
621	{
622	/*
623	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
624	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
625	*/
626	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
627	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
628	* and @bugref{6944}. */
629	PVM pVM = pVCpu->CTX_SUFF(pVM);
630	return ( X86_CR0_PE
631	\| X86_CR0_NE
632	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
633	\| X86_CR0_PG
634	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
635	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
636	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
637	}
638
639
640	/**
641	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
642	*
643	* Any bit set in this mask is owned by the host/hypervisor and would cause a
644	* VM-exit when modified by the guest.
645	*
646	* @returns The static CR4 guest/host mask.
647	* @param pVCpu The cross context virtual CPU structure.
648	*/
649	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
650	{
651	/*
652	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
653	* these bits are reserved on hardware that does not support them. Since the
654	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
655	* these bits and handle it depending on whether we expose them to the guest.
656	*/
657	PVM pVM = pVCpu->CTX_SUFF(pVM);
658	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
659	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
660	return ( X86_CR4_VMXE
661	\| X86_CR4_VME
662	\| X86_CR4_PAE
663	\| X86_CR4_PGE
664	\| X86_CR4_PSE
665	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
666	\| (fPcid ? X86_CR4_PCIDE : 0));
667	}
668
669
670	/**
671	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
672	* area.
673	*
674	* @returns @c true if it's different, @c false otherwise.
675	* @param pVmcsInfo The VMCS info. object.
676	*/
677	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
678	{
679	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
680	&& pVmcsInfo->pvGuestMsrStore);
681	}
682
683
684	/**
685	* Adds one or more exceptions to the exception bitmap and commits it to the current
686	* VMCS.
687	*
688	* @returns VBox status code.
689	* @param pVmxTransient The VMX-transient structure.
690	* @param uXcptMask The exception(s) to add.
691	*/
692	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
693	{
694	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
695	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
696	if ((uXcptBitmap & uXcptMask) != uXcptMask)
697	{
698	uXcptBitmap \|= uXcptMask;
699	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
700	AssertRCReturn(rc, rc);
701	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
702	}
703	return VINF_SUCCESS;
704	}
705
706
707	/**
708	* Adds an exception to the exception bitmap and commits it to the current VMCS.
709	*
710	* @returns VBox status code.
711	* @param pVmxTransient The VMX-transient structure.
712	* @param uXcpt The exception to add.
713	*/
714	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
715	{
716	Assert(uXcpt <= X86_XCPT_LAST);
717	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
718	}
719
720
721	/**
722	* Remove one or more exceptions from the exception bitmap and commits it to the
723	* current VMCS.
724	*
725	* This takes care of not removing the exception intercept if a nested-guest
726	* requires the exception to be intercepted.
727	*
728	* @returns VBox status code.
729	* @param pVCpu The cross context virtual CPU structure.
730	* @param pVmxTransient The VMX-transient structure.
731	* @param uXcptMask The exception(s) to remove.
732	*/
733	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
734	{
735	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
736	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
737	if (u32XcptBitmap & uXcptMask)
738	{
739	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
740	if (!pVmxTransient->fIsNestedGuest)
741	{ /* likely */ }
742	else
743	{
744	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
745	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
746	}
747	#endif
748	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
749	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
750	\| RT_BIT(X86_XCPT_DE)
751	\| RT_BIT(X86_XCPT_NM)
752	\| RT_BIT(X86_XCPT_TS)
753	\| RT_BIT(X86_XCPT_UD)
754	\| RT_BIT(X86_XCPT_NP)
755	\| RT_BIT(X86_XCPT_SS)
756	\| RT_BIT(X86_XCPT_GP)
757	\| RT_BIT(X86_XCPT_PF)
758	\| RT_BIT(X86_XCPT_MF));
759	#elif defined(HMVMX_ALWAYS_TRAP_PF)
760	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
761	#endif
762	if (uXcptMask)
763	{
764	/* Validate we are not removing any essential exception intercepts. */
765	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
766	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
767	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
768
769	/* Remove it from the exception bitmap. */
770	u32XcptBitmap &= ~uXcptMask;
771
772	/* Commit and update the cache if necessary. */
773	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
774	{
775	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
776	AssertRCReturn(rc, rc);
777	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
778	}
779	}
780	}
781	return VINF_SUCCESS;
782	}
783
784
785	/**
786	* Remove an exceptions from the exception bitmap and commits it to the current
787	* VMCS.
788	*
789	* @returns VBox status code.
790	* @param pVCpu The cross context virtual CPU structure.
791	* @param pVmxTransient The VMX-transient structure.
792	* @param uXcpt The exception to remove.
793	*/
794	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
795	{
796	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
797	}
798
799
800	/**
801	* Loads the VMCS specified by the VMCS info. object.
802	*
803	* @returns VBox status code.
804	* @param pVmcsInfo The VMCS info. object.
805	*/
806	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
807	{
808	Assert(pVmcsInfo);
809	Assert(pVmcsInfo->HCPhysVmcs);
810	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
811
812	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
813	{
814	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
815	if (RT_SUCCESS(rc))
816	{
817	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
818	return VINF_SUCCESS;
819	}
820	return rc;
821	}
822	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
823	}
824
825
826	/**
827	* Clears the VMCS specified by the VMCS info. object.
828	*
829	* @returns VBox status code.
830	* @param pVmcsInfo The VMCS info. object.
831	*/
832	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
833	{
834	Assert(pVmcsInfo);
835	Assert(pVmcsInfo->HCPhysVmcs);
836	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
837
838	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
839	if (RT_SUCCESS(rc))
840	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
841	return rc;
842	}
843
844
845	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
846	/**
847	* Switches the current VMCS to the one specified.
848	*
849	* @returns VBox status code.
850	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
851	* @param pVmcsInfoTo The VMCS info. object we are switching to.
852	*
853	* @remarks Called with interrupts disabled.
854	*/
855	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
856	{
857	Assert(pVmcsInfoFrom);
858	Assert(pVmcsInfoTo);
859
860	/*
861	* Clear the VMCS we are switching out if it has not already been cleared.
862	* This will sync any CPU internal data back to the VMCS.
863	*/
864	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
865	{
866	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
867	if (RT_SUCCESS(rc))
868	{ /* likely */ }
869	else
870	return rc;
871	}
872
873	/*
874	* Clear the VMCS we are switching to if it has not already been cleared.
875	* This will initialize the VMCS launch state to "clear" required for loading it.
876	*
877	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
878	*/
879	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
880	{
881	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
882	if (RT_SUCCESS(rc))
883	{ /* likely */ }
884	else
885	return rc;
886	}
887
888	/*
889	* Finally, load the VMCS we are switching to.
890	*/
891	return hmR0VmxLoadVmcs(pVmcsInfoTo);
892	}
893	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
894
895
896	/**
897	* Updates the VM's last error record.
898	*
899	* If there was a VMX instruction error, reads the error data from the VMCS and
900	* updates VCPU's last error record as well.
901	*
902	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
903	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
904	* VERR_VMX_INVALID_VMCS_FIELD.
905	* @param rc The error code.
906	*/
907	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
908	{
909	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
910	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
911	{
912	AssertPtrReturnVoid(pVCpu);
913	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
914	}
915	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
916	}
917
918
919	#ifdef VBOX_STRICT
920	/**
921	* Reads the VM-entry interruption-information field from the VMCS into the VMX
922	* transient structure.
923	*
924	* @returns VBox status code.
925	* @param pVmxTransient The VMX-transient structure.
926	*
927	* @remarks No-long-jump zone!!!
928	*/
929	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
930	{
931	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
932	AssertRCReturn(rc, rc);
933	return VINF_SUCCESS;
934	}
935
936
937	/**
938	* Reads the VM-entry exception error code field from the VMCS into
939	* the VMX transient structure.
940	*
941	* @returns VBox status code.
942	* @param pVmxTransient The VMX-transient structure.
943	*
944	* @remarks No-long-jump zone!!!
945	*/
946	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
947	{
948	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
949	AssertRCReturn(rc, rc);
950	return VINF_SUCCESS;
951	}
952
953
954	/**
955	* Reads the VM-entry exception error code field from the VMCS into
956	* the VMX transient structure.
957	*
958	* @returns VBox status code.
959	* @param pVmxTransient The VMX-transient structure.
960	*
961	* @remarks No-long-jump zone!!!
962	*/
963	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
964	{
965	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
966	AssertRCReturn(rc, rc);
967	return VINF_SUCCESS;
968	}
969	#endif /* VBOX_STRICT */
970
971
972	/**
973	* Reads the VM-exit interruption-information field from the VMCS into the VMX
974	* transient structure.
975	*
976	* @returns VBox status code.
977	* @param pVmxTransient The VMX-transient structure.
978	*/
979	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
980	{
981	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
982	{
983	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
984	AssertRCReturn(rc,rc);
985	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
986	}
987	return VINF_SUCCESS;
988	}
989
990
991	/**
992	* Reads the VM-exit interruption error code from the VMCS into the VMX
993	* transient structure.
994	*
995	* @returns VBox status code.
996	* @param pVmxTransient The VMX-transient structure.
997	*/
998	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
999	{
1000	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1001	{
1002	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1003	AssertRCReturn(rc, rc);
1004	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1005	}
1006	return VINF_SUCCESS;
1007	}
1008
1009
1010	/**
1011	* Reads the VM-exit instruction length field from the VMCS into the VMX
1012	* transient structure.
1013	*
1014	* @returns VBox status code.
1015	* @param pVmxTransient The VMX-transient structure.
1016	*/
1017	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1018	{
1019	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1020	{
1021	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1022	AssertRCReturn(rc, rc);
1023	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1024	}
1025	return VINF_SUCCESS;
1026	}
1027
1028
1029	/**
1030	* Reads the VM-exit instruction-information field from the VMCS into
1031	* the VMX transient structure.
1032	*
1033	* @returns VBox status code.
1034	* @param pVmxTransient The VMX-transient structure.
1035	*/
1036	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1037	{
1038	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1039	{
1040	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1041	AssertRCReturn(rc, rc);
1042	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1043	}
1044	return VINF_SUCCESS;
1045	}
1046
1047
1048	/**
1049	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1050	*
1051	* @returns VBox status code.
1052	* @param pVCpu The cross context virtual CPU structure of the
1053	* calling EMT. (Required for the VMCS cache case.)
1054	* @param pVmxTransient The VMX-transient structure.
1055	*/
1056	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1057	{
1058	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1059	{
1060	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1061	AssertRCReturn(rc, rc);
1062	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1063	}
1064	return VINF_SUCCESS;
1065	}
1066
1067
1068	/**
1069	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1070	*
1071	* @returns VBox status code.
1072	* @param pVCpu The cross context virtual CPU structure of the
1073	* calling EMT. (Required for the VMCS cache case.)
1074	* @param pVmxTransient The VMX-transient structure.
1075	*/
1076	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1077	{
1078	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1079	{
1080	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1081	AssertRCReturn(rc, rc);
1082	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1083	}
1084	return VINF_SUCCESS;
1085	}
1086
1087
1088	/**
1089	* Reads the IDT-vectoring information field from the VMCS into the VMX
1090	* transient structure.
1091	*
1092	* @returns VBox status code.
1093	* @param pVmxTransient The VMX-transient structure.
1094	*
1095	* @remarks No-long-jump zone!!!
1096	*/
1097	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1098	{
1099	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1100	{
1101	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1102	AssertRCReturn(rc, rc);
1103	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1104	}
1105	return VINF_SUCCESS;
1106	}
1107
1108
1109	/**
1110	* Reads the IDT-vectoring error code from the VMCS into the VMX
1111	* transient structure.
1112	*
1113	* @returns VBox status code.
1114	* @param pVmxTransient The VMX-transient structure.
1115	*/
1116	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1117	{
1118	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1119	{
1120	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1121	AssertRCReturn(rc, rc);
1122	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1123	}
1124	return VINF_SUCCESS;
1125	}
1126
1127
1128	/**
1129	* Enters VMX root mode operation on the current CPU.
1130	*
1131	* @returns VBox status code.
1132	* @param pVM The cross context VM structure. Can be
1133	* NULL, after a resume.
1134	* @param HCPhysCpuPage Physical address of the VMXON region.
1135	* @param pvCpuPage Pointer to the VMXON region.
1136	*/
1137	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1138	{
1139	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1140	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1141	Assert(pvCpuPage);
1142	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1143
1144	if (pVM)
1145	{
1146	/* Write the VMCS revision identifier to the VMXON region. */
1147	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1148	}
1149
1150	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1151	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1152
1153	/* Enable the VMX bit in CR4 if necessary. */
1154	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1155
1156	/* Enter VMX root mode. */
1157	int rc = VMXEnable(HCPhysCpuPage);
1158	if (RT_FAILURE(rc))
1159	{
1160	if (!(uOldCr4 & X86_CR4_VMXE))
1161	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1162
1163	if (pVM)
1164	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1165	}
1166
1167	/* Restore interrupts. */
1168	ASMSetFlags(fEFlags);
1169	return rc;
1170	}
1171
1172
1173	/**
1174	* Exits VMX root mode operation on the current CPU.
1175	*
1176	* @returns VBox status code.
1177	*/
1178	static int hmR0VmxLeaveRootMode(void)
1179	{
1180	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1181
1182	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1183	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1184
1185	/* If we're for some reason not in VMX root mode, then don't leave it. */
1186	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1187
1188	int rc;
1189	if (uHostCR4 & X86_CR4_VMXE)
1190	{
1191	/* Exit VMX root mode and clear the VMX bit in CR4. */
1192	VMXDisable();
1193	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1194	rc = VINF_SUCCESS;
1195	}
1196	else
1197	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1198
1199	/* Restore interrupts. */
1200	ASMSetFlags(fEFlags);
1201	return rc;
1202	}
1203
1204
1205	/**
1206	* Allocates and maps a physically contiguous page. The allocated page is
1207	* zero'd out (used by various VT-x structures).
1208	*
1209	* @returns IPRT status code.
1210	* @param pMemObj Pointer to the ring-0 memory object.
1211	* @param ppVirt Where to store the virtual address of the
1212	* allocation.
1213	* @param pHCPhys Where to store the physical address of the
1214	* allocation.
1215	*/
1216	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1217	{
1218	AssertPtr(pMemObj);
1219	AssertPtr(ppVirt);
1220	AssertPtr(pHCPhys);
1221	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1222	if (RT_FAILURE(rc))
1223	return rc;
1224	ppVirt = RTR0MemObjAddress(pMemObj);
1225	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1226	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1227	return VINF_SUCCESS;
1228	}
1229
1230
1231	/**
1232	* Frees and unmaps an allocated, physical page.
1233	*
1234	* @param pMemObj Pointer to the ring-0 memory object.
1235	* @param ppVirt Where to re-initialize the virtual address of
1236	* allocation as 0.
1237	* @param pHCPhys Where to re-initialize the physical address of the
1238	* allocation as 0.
1239	*/
1240	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1241	{
1242	AssertPtr(pMemObj);
1243	AssertPtr(ppVirt);
1244	AssertPtr(pHCPhys);
1245	/* NULL is valid, accepted and ignored by the free function below. */
1246	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1247	*pMemObj = NIL_RTR0MEMOBJ;
1248	*ppVirt = NULL;
1249	*pHCPhys = NIL_RTHCPHYS;
1250	}
1251
1252
1253	/**
1254	* Initializes a VMCS info. object.
1255	*
1256	* @param pVmcsInfo The VMCS info. object.
1257	*/
1258	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1259	{
1260	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1261
1262	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1265	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1266	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1267	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1272	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1273	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1274	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1275	}
1276
1277
1278	/**
1279	* Frees the VT-x structures for a VMCS info. object.
1280	*
1281	* @param pVM The cross context VM structure.
1282	* @param pVmcsInfo The VMCS info. object.
1283	*/
1284	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1285	{
1286	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1287
1288	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1290
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1292	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1293	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1294
1295	hmR0VmxInitVmcsInfo(pVmcsInfo);
1296	}
1297
1298
1299	/**
1300	* Allocates the VT-x structures for a VMCS info. object.
1301	*
1302	* @returns VBox status code.
1303	* @param pVCpu The cross context virtual CPU structure.
1304	* @param pVmcsInfo The VMCS info. object.
1305	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1306	*/
1307	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1308	{
1309	PVM pVM = pVCpu->CTX_SUFF(pVM);
1310
1311	/* Allocate the guest VM control structure (VMCS). */
1312	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1313	if (RT_SUCCESS(rc))
1314	{
1315	if (!fIsNstGstVmcs)
1316	{
1317	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1318	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1319	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1320	{
1321	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1322	NULL /* pR3Ptr /, NULL / pRCPtr */);
1323	}
1324	}
1325	else
1326	{
1327	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1328	Assert(!pVmcsInfo->pbVirtApic);
1329	}
1330
1331	if (RT_SUCCESS(rc))
1332	{
1333	/*
1334	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1335	* transparent accesses of specific MSRs.
1336	*
1337	* If the condition for enabling MSR bitmaps changes here, don't forget to
1338	* update HMIsMsrBitmapActive().
1339	*
1340	* We don't share MSR bitmaps between the guest and nested-guest as we then
1341	* don't need to care about carefully restoring the guest MSR bitmap.
1342	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1343	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1344	*/
1345	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1346	{
1347	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1348	if (RT_SUCCESS(rc))
1349	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1350	}
1351
1352	if (RT_SUCCESS(rc))
1353	{
1354	/*
1355	* Allocate the VM-entry MSR-load area for the guest MSRs.
1356	*
1357	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1358	* the guest and nested-guest.
1359	*/
1360	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1361	&pVmcsInfo->HCPhysGuestMsrLoad);
1362	if (RT_SUCCESS(rc))
1363	{
1364	/*
1365	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1366	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1367	*/
1368	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1369	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1370	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1371
1372	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1373	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1374	&pVmcsInfo->HCPhysHostMsrLoad);
1375	}
1376	}
1377	}
1378	}
1379
1380	return rc;
1381	}
1382
1383
1384	/**
1385	* Free all VT-x structures for the VM.
1386	*
1387	* @returns IPRT status code.
1388	* @param pVM The cross context VM structure.
1389	*/
1390	static void hmR0VmxStructsFree(PVM pVM)
1391	{
1392	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1394	#endif
1395	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1396
1397	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1398	{
1399	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1400	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1401	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1402	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1403	if (pVM->cpum.ro.GuestFeatures.fVmx)
1404	{
1405	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1406	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1407	}
1408	#endif
1409	}
1410	}
1411
1412
1413	/**
1414	* Allocate all VT-x structures for the VM.
1415	*
1416	* @returns IPRT status code.
1417	* @param pVM The cross context VM structure.
1418	*/
1419	static int hmR0VmxStructsAlloc(PVM pVM)
1420	{
1421	/*
1422	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1423	* The VMCS size cannot be more than 4096 bytes.
1424	*
1425	* See Intel spec. Appendix A.1 "Basic VMX Information".
1426	*/
1427	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1428	if (cbVmcs <= X86_PAGE_4K_SIZE)
1429	{ /* likely */ }
1430	else
1431	{
1432	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1433	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1434	}
1435
1436	/*
1437	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1438	*/
1439	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1440	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1441	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1442	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1443	#endif
1444
1445	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1446	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1447	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1448
1449	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1450	{
1451	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1452	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1453	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1454	}
1455
1456	/*
1457	* Allocate per-VM VT-x structures.
1458	*/
1459	int rc = VINF_SUCCESS;
1460	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1461	/* Allocate crash-dump magic scratch page. */
1462	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1463	if (RT_FAILURE(rc))
1464	{
1465	hmR0VmxStructsFree(pVM);
1466	return rc;
1467	}
1468	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1469	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1470	#endif
1471
1472	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1473	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1474	{
1475	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1476	&pVM->hm.s.vmx.HCPhysApicAccess);
1477	if (RT_FAILURE(rc))
1478	{
1479	hmR0VmxStructsFree(pVM);
1480	return rc;
1481	}
1482	}
1483
1484	/*
1485	* Initialize per-VCPU VT-x structures.
1486	*/
1487	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1488	{
1489	/* Allocate the guest VMCS structures. */
1490	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1491	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1492	if (RT_SUCCESS(rc))
1493	{
1494	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1495	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1496	if (pVM->cpum.ro.GuestFeatures.fVmx)
1497	{
1498	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1499	if (RT_SUCCESS(rc))
1500	{ /* likely */ }
1501	else
1502	break;
1503	}
1504	#endif
1505	}
1506	else
1507	break;
1508	}
1509
1510	if (RT_FAILURE(rc))
1511	{
1512	hmR0VmxStructsFree(pVM);
1513	return rc;
1514	}
1515
1516	return VINF_SUCCESS;
1517	}
1518
1519
1520	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1521	/**
1522	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1523	*
1524	* @returns @c true if the MSR is intercepted, @c false otherwise.
1525	* @param pvMsrBitmap The MSR bitmap.
1526	* @param offMsr The MSR byte offset.
1527	* @param iBit The bit offset from the byte offset.
1528	*/
1529	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1530	{
1531	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1532	Assert(pbMsrBitmap);
1533	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1534	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1535	}
1536	#endif
1537
1538
1539	/**
1540	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1541	*
1542	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1543	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1544	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1545	* the read/write access of this MSR.
1546	*
1547	* @param pVCpu The cross context virtual CPU structure.
1548	* @param pVmcsInfo The VMCS info. object.
1549	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1550	* @param idMsr The MSR value.
1551	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1552	* include both a read -and- a write permission!
1553	*
1554	* @sa CPUMGetVmxMsrPermission.
1555	* @remarks Can be called with interrupts disabled.
1556	*/
1557	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1558	{
1559	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1560	Assert(pbMsrBitmap);
1561	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1562
1563	/*
1564	* MSR-bitmap Layout:
1565	* Byte index MSR range Interpreted as
1566	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1567	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1568	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1569	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1570	*
1571	* A bit corresponding to an MSR within the above range causes a VM-exit
1572	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1573	* the MSR range, it always cause a VM-exit.
1574	*
1575	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1576	*/
1577	uint16_t const offBitmapRead = 0;
1578	uint16_t const offBitmapWrite = 0x800;
1579	uint16_t offMsr;
1580	int32_t iBit;
1581	if (idMsr <= UINT32_C(0x00001fff))
1582	{
1583	offMsr = 0;
1584	iBit = idMsr;
1585	}
1586	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1587	{
1588	offMsr = 0x400;
1589	iBit = idMsr - UINT32_C(0xc0000000);
1590	}
1591	else
1592	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1593
1594	/*
1595	* Set the MSR read permission.
1596	*/
1597	uint16_t const offMsrRead = offBitmapRead + offMsr;
1598	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1599	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1600	{
1601	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1602	bool const fClear = !fIsNstGstVmcs ? true
1603	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1604	#else
1605	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1606	bool const fClear = true;
1607	#endif
1608	if (fClear)
1609	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1610	}
1611	else
1612	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1613
1614	/*
1615	* Set the MSR write permission.
1616	*/
1617	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1618	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1619	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1620	{
1621	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1622	bool const fClear = !fIsNstGstVmcs ? true
1623	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1624	#else
1625	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1626	bool const fClear = true;
1627	#endif
1628	if (fClear)
1629	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631	else
1632	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1633	}
1634
1635
1636	/**
1637	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1638	* area.
1639	*
1640	* @returns VBox status code.
1641	* @param pVCpu The cross context virtual CPU structure.
1642	* @param pVmcsInfo The VMCS info. object.
1643	* @param cMsrs The number of MSRs.
1644	*/
1645	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1646	{
1647	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1648	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1649	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1650	{
1651	/* Commit the MSR counts to the VMCS and update the cache. */
1652	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1653	{
1654	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1655	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1657	AssertRCReturn(rc, rc);
1658
1659	pVmcsInfo->cEntryMsrLoad = cMsrs;
1660	pVmcsInfo->cExitMsrStore = cMsrs;
1661	pVmcsInfo->cExitMsrLoad = cMsrs;
1662	}
1663	return VINF_SUCCESS;
1664	}
1665
1666	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1667	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1668	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1669	}
1670
1671
1672	/**
1673	* Adds a new (or updates the value of an existing) guest/host MSR
1674	* pair to be swapped during the world-switch as part of the
1675	* auto-load/store MSR area in the VMCS.
1676	*
1677	* @returns VBox status code.
1678	* @param pVCpu The cross context virtual CPU structure.
1679	* @param pVmxTransient The VMX-transient structure.
1680	* @param idMsr The MSR.
1681	* @param uGuestMsrValue Value of the guest MSR.
1682	* @param fSetReadWrite Whether to set the guest read/write access of this
1683	* MSR (thus not causing a VM-exit).
1684	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1685	* necessary.
1686	*/
1687	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1688	bool fSetReadWrite, bool fUpdateHostMsr)
1689	{
1690	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1691	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1692	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1693	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1694	uint32_t i;
1695
1696	/* Paranoia. */
1697	Assert(pGuestMsrLoad);
1698
1699	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1700
1701	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1702	for (i = 0; i < cMsrs; i++)
1703	{
1704	if (pGuestMsrLoad[i].u32Msr == idMsr)
1705	break;
1706	}
1707
1708	bool fAdded = false;
1709	if (i == cMsrs)
1710	{
1711	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1712	++cMsrs;
1713	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1714	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1715
1716	/* Set the guest to read/write this MSR without causing VM-exits. */
1717	if ( fSetReadWrite
1718	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1719	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1720
1721	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1722	fAdded = true;
1723	}
1724
1725	/* Update the MSR value for the newly added or already existing MSR. */
1726	pGuestMsrLoad[i].u32Msr = idMsr;
1727	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1728
1729	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1730	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1731	{
1732	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1733	pGuestMsrStore[i].u32Msr = idMsr;
1734	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1735	}
1736
1737	/* Update the corresponding slot in the host MSR area. */
1738	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1739	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1740	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1741	pHostMsr[i].u32Msr = idMsr;
1742
1743	/*
1744	* Only if the caller requests to update the host MSR value AND we've newly added the
1745	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1746	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1747	*
1748	* We do this for performance reasons since reading MSRs may be quite expensive.
1749	*/
1750	if (fAdded)
1751	{
1752	if (fUpdateHostMsr)
1753	{
1754	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1755	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1756	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1757	}
1758	else
1759	{
1760	/* Someone else can do the work. */
1761	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1762	}
1763	}
1764	return VINF_SUCCESS;
1765	}
1766
1767
1768	/**
1769	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1770	* auto-load/store MSR area in the VMCS.
1771	*
1772	* @returns VBox status code.
1773	* @param pVCpu The cross context virtual CPU structure.
1774	* @param pVmxTransient The VMX-transient structure.
1775	* @param idMsr The MSR.
1776	*/
1777	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1778	{
1779	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1780	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1781	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1782	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1783
1784	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1785
1786	for (uint32_t i = 0; i < cMsrs; i++)
1787	{
1788	/* Find the MSR. */
1789	if (pGuestMsrLoad[i].u32Msr == idMsr)
1790	{
1791	/*
1792	* If it's the last MSR, we only need to reduce the MSR count.
1793	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1794	*/
1795	if (i < cMsrs - 1)
1796	{
1797	/* Remove it from the VM-entry MSR-load area. */
1798	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1799	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1800
1801	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1802	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1803	{
1804	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1805	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1806	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1807	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1808	}
1809
1810	/* Remove it from the VM-exit MSR-load area. */
1811	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1812	Assert(pHostMsr[i].u32Msr == idMsr);
1813	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1814	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1815	}
1816
1817	/* Reduce the count to reflect the removed MSR and bail. */
1818	--cMsrs;
1819	break;
1820	}
1821	}
1822
1823	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1824	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1825	{
1826	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1827	AssertRCReturn(rc, rc);
1828
1829	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1830	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1831	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1832
1833	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1834	return VINF_SUCCESS;
1835	}
1836
1837	return VERR_NOT_FOUND;
1838	}
1839
1840
1841	/**
1842	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1843	*
1844	* @returns @c true if found, @c false otherwise.
1845	* @param pVmcsInfo The VMCS info. object.
1846	* @param idMsr The MSR to find.
1847	*/
1848	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1849	{
1850	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1851	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1852	Assert(pMsrs);
1853	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1854	for (uint32_t i = 0; i < cMsrs; i++)
1855	{
1856	if (pMsrs[i].u32Msr == idMsr)
1857	return true;
1858	}
1859	return false;
1860	}
1861
1862
1863	/**
1864	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1865	*
1866	* @param pVCpu The cross context virtual CPU structure.
1867	* @param pVmcsInfo The VMCS info. object.
1868	*
1869	* @remarks No-long-jump zone!!!
1870	*/
1871	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1872	{
1873	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1874
1875	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1876	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1877	Assert(pHostMsrLoad);
1878	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1879	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1880	for (uint32_t i = 0; i < cMsrs; i++)
1881	{
1882	/*
1883	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1884	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1885	*/
1886	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1887	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1888	else
1889	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1890	}
1891	}
1892
1893
1894	/**
1895	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1896	* perform lazy restoration of the host MSRs while leaving VT-x.
1897	*
1898	* @param pVCpu The cross context virtual CPU structure.
1899	*
1900	* @remarks No-long-jump zone!!!
1901	*/
1902	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1903	{
1904	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1905
1906	/*
1907	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1908	*/
1909	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1910	{
1911	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1912	#if HC_ARCH_BITS == 64
1913	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1914	{
1915	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1916	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1917	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1918	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1919	}
1920	#endif
1921	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1922	}
1923	}
1924
1925
1926	/**
1927	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1928	* lazily while leaving VT-x.
1929	*
1930	* @returns true if it does, false otherwise.
1931	* @param pVCpu The cross context virtual CPU structure.
1932	* @param idMsr The MSR to check.
1933	*/
1934	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1935	{
1936	NOREF(pVCpu);
1937	#if HC_ARCH_BITS == 64
1938	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1939	{
1940	switch (idMsr)
1941	{
1942	case MSR_K8_LSTAR:
1943	case MSR_K6_STAR:
1944	case MSR_K8_SF_MASK:
1945	case MSR_K8_KERNEL_GS_BASE:
1946	return true;
1947	}
1948	}
1949	#else
1950	RT_NOREF(pVCpu, idMsr);
1951	#endif
1952	return false;
1953	}
1954
1955
1956	/**
1957	* Loads a set of guests MSRs to allow read/passthru to the guest.
1958	*
1959	* The name of this function is slightly confusing. This function does NOT
1960	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1961	* common prefix for functions dealing with "lazy restoration" of the shared
1962	* MSRs.
1963	*
1964	* @param pVCpu The cross context virtual CPU structure.
1965	*
1966	* @remarks No-long-jump zone!!!
1967	*/
1968	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1969	{
1970	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1971	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1972
1973	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1974	#if HC_ARCH_BITS == 64
1975	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1976	{
1977	/*
1978	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1979	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1980	* we can skip a few MSR writes.
1981	*
1982	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1983	* guest MSR values in the guest-CPU context might be different to what's currently
1984	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1985	* CPU, see @bugref{8728}.
1986	*/
1987	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1988	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1989	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1990	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1991	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1992	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1993	{
1994	#ifdef VBOX_STRICT
1995	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1996	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1997	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1998	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1999	#endif
2000	}
2001	else
2002	{
2003	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2004	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2005	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2006	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2007	}
2008	}
2009	#endif
2010	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2011	}
2012
2013
2014	/**
2015	* Performs lazy restoration of the set of host MSRs if they were previously
2016	* loaded with guest MSR values.
2017	*
2018	* @param pVCpu The cross context virtual CPU structure.
2019	*
2020	* @remarks No-long-jump zone!!!
2021	* @remarks The guest MSRs should have been saved back into the guest-CPU
2022	* context by hmR0VmxImportGuestState()!!!
2023	*/
2024	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2025	{
2026	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2027	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2028
2029	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2030	{
2031	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2032	#if HC_ARCH_BITS == 64
2033	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2034	{
2035	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2036	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2037	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2038	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2039	}
2040	#endif
2041	}
2042	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2043	}
2044
2045
2046	/**
2047	* Verifies that our cached values of the VMCS fields are all consistent with
2048	* what's actually present in the VMCS.
2049	*
2050	* @returns VBox status code.
2051	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2052	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2053	* VMCS content. HMCPU error-field is
2054	* updated, see VMX_VCI_XXX.
2055	* @param pVCpu The cross context virtual CPU structure.
2056	* @param pVmcsInfo The VMCS info. object.
2057	*/
2058	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2059	{
2060	uint32_t u32Val;
2061	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2062	AssertRCReturn(rc, rc);
2063	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2064	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2065	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2066	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2067
2068	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2069	AssertRCReturn(rc, rc);
2070	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2071	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2072	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2073	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2074
2075	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2076	AssertRCReturn(rc, rc);
2077	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2078	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2079	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2080	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2081
2082	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2083	AssertRCReturn(rc, rc);
2084	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2085	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2086	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2087	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2088
2089	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2090	{
2091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2092	AssertRCReturn(rc, rc);
2093	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2094	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2095	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2096	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2097	}
2098
2099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2100	AssertRCReturn(rc, rc);
2101	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2102	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2103	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2104	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2105
2106	uint64_t u64Val;
2107	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2108	AssertRCReturn(rc, rc);
2109	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2110	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2111	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2112	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2113
2114	return VINF_SUCCESS;
2115	}
2116
2117
2118	#ifdef VBOX_STRICT
2119	/**
2120	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2121	*
2122	* @param pVCpu The cross context virtual CPU structure.
2123	* @param pVmcsInfo The VMCS info. object.
2124	*/
2125	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2126	{
2127	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2128
2129	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2130	{
2131	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2132	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2133	uint64_t uVmcsEferMsrVmcs;
2134	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2135	AssertRC(rc);
2136
2137	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2138	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2139	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2140	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2141	}
2142	}
2143
2144
2145	/**
2146	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2147	* VMCS are correct.
2148	*
2149	* @param pVCpu The cross context virtual CPU structure.
2150	* @param pVmcsInfo The VMCS info. object.
2151	*/
2152	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2153	{
2154	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2155
2156	/* Read the various MSR-area counts from the VMCS. */
2157	uint32_t cEntryLoadMsrs;
2158	uint32_t cExitStoreMsrs;
2159	uint32_t cExitLoadMsrs;
2160	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2162	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2163
2164	/* Verify all the MSR counts are the same. */
2165	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2166	Assert(cExitStoreMsrs == cExitLoadMsrs);
2167	uint32_t const cMsrs = cExitLoadMsrs;
2168
2169	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2170	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2171
2172	/* Verify the MSR counts are within the allocated page size. */
2173	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2174
2175	/* Verify the relevant contents of the MSR areas match. */
2176	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2177	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2178	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2179	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2180	for (uint32_t i = 0; i < cMsrs; i++)
2181	{
2182	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2183	if (fSeparateExitMsrStorePage)
2184	{
2185	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2186	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2187	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2188	}
2189
2190	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2191	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2192	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2193
2194	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2195	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2196	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2197	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2198
2199	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2200	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2201	if (fIsEferMsr)
2202	{
2203	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2204	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2205	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2206	}
2207
2208	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2209	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2210	{
2211	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2212	if (fIsEferMsr)
2213	{
2214	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2215	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2216	}
2217	else
2218	{
2219	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2220	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2221	}
2222	}
2223
2224	/* Move to the next MSR. */
2225	pHostMsrLoad++;
2226	pGuestMsrLoad++;
2227	pGuestMsrStore++;
2228	}
2229	}
2230	#endif /* VBOX_STRICT */
2231
2232
2233	/**
2234	* Flushes the TLB using EPT.
2235	*
2236	* @returns VBox status code.
2237	* @param pVCpu The cross context virtual CPU structure of the calling
2238	* EMT. Can be NULL depending on @a enmTlbFlush.
2239	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2240	* enmTlbFlush.
2241	* @param enmTlbFlush Type of flush.
2242	*
2243	* @remarks Caller is responsible for making sure this function is called only
2244	* when NestedPaging is supported and providing @a enmTlbFlush that is
2245	* supported by the CPU.
2246	* @remarks Can be called with interrupts disabled.
2247	*/
2248	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2249	{
2250	uint64_t au64Descriptor[2];
2251	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2252	au64Descriptor[0] = 0;
2253	else
2254	{
2255	Assert(pVCpu);
2256	Assert(pVmcsInfo);
2257	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2258	}
2259	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2260
2261	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2262	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2263
2264	if ( RT_SUCCESS(rc)
2265	&& pVCpu)
2266	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2267	}
2268
2269
2270	/**
2271	* Flushes the TLB using VPID.
2272	*
2273	* @returns VBox status code.
2274	* @param pVCpu The cross context virtual CPU structure of the calling
2275	* EMT. Can be NULL depending on @a enmTlbFlush.
2276	* @param enmTlbFlush Type of flush.
2277	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2278	* on @a enmTlbFlush).
2279	*
2280	* @remarks Can be called with interrupts disabled.
2281	*/
2282	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2283	{
2284	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2285
2286	uint64_t au64Descriptor[2];
2287	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2288	{
2289	au64Descriptor[0] = 0;
2290	au64Descriptor[1] = 0;
2291	}
2292	else
2293	{
2294	AssertPtr(pVCpu);
2295	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2296	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2297	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2298	au64Descriptor[1] = GCPtr;
2299	}
2300
2301	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2302	AssertMsg(rc == VINF_SUCCESS,
2303	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2304
2305	if ( RT_SUCCESS(rc)
2306	&& pVCpu)
2307	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2308	NOREF(rc);
2309	}
2310
2311
2312	/**
2313	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2314	* otherwise there is nothing really to invalidate.
2315	*
2316	* @returns VBox status code.
2317	* @param pVCpu The cross context virtual CPU structure.
2318	* @param GCVirt Guest virtual address of the page to invalidate.
2319	*/
2320	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2321	{
2322	AssertPtr(pVCpu);
2323	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2324
2325	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2326	{
2327	/*
2328	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2329	* the EPT case. See @bugref{6043} and @bugref{6177}.
2330	*
2331	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2332	* as this function maybe called in a loop with individual addresses.
2333	*/
2334	PVM pVM = pVCpu->CTX_SUFF(pVM);
2335	if (pVM->hm.s.vmx.fVpid)
2336	{
2337	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2338
2339	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2340	/*
2341	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2342	* where executing INVVPID outside 64-bit mode does not flush translations of
2343	* 64-bit linear addresses, see @bugref{6208#c72}.
2344	*/
2345	if (RT_HI_U32(GCVirt))
2346	fVpidFlush = false;
2347	#endif
2348
2349	if (fVpidFlush)
2350	{
2351	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2352	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2353	}
2354	else
2355	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2356	}
2357	else if (pVM->hm.s.fNestedPaging)
2358	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2359	}
2360
2361	return VINF_SUCCESS;
2362	}
2363
2364
2365	/**
2366	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2367	* case where neither EPT nor VPID is supported by the CPU.
2368	*
2369	* @param pHostCpu The HM physical-CPU structure.
2370	* @param pVCpu The cross context virtual CPU structure.
2371	*
2372	* @remarks Called with interrupts disabled.
2373	*/
2374	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2375	{
2376	AssertPtr(pVCpu);
2377	AssertPtr(pHostCpu);
2378
2379	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2380
2381	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2382	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2383	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2384	pVCpu->hm.s.fForceTLBFlush = false;
2385	return;
2386	}
2387
2388
2389	/**
2390	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2391	*
2392	* @param pHostCpu The HM physical-CPU structure.
2393	* @param pVCpu The cross context virtual CPU structure.
2394	* @param pVmcsInfo The VMCS info. object.
2395	*
2396	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2397	* nomenclature. The reason is, to avoid confusion in compare statements
2398	* since the host-CPU copies are named "ASID".
2399	*
2400	* @remarks Called with interrupts disabled.
2401	*/
2402	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2403	{
2404	#ifdef VBOX_WITH_STATISTICS
2405	bool fTlbFlushed = false;
2406	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2407	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2408	if (!fTlbFlushed) \
2409	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2410	} while (0)
2411	#else
2412	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2413	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2414	#endif
2415
2416	AssertPtr(pVCpu);
2417	AssertPtr(pHostCpu);
2418	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2419
2420	PVM pVM = pVCpu->CTX_SUFF(pVM);
2421	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2422	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2423	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2424
2425	/*
2426	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2427	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2428	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2429	* cannot reuse the current ASID anymore.
2430	*/
2431	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2432	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2433	{
2434	++pHostCpu->uCurrentAsid;
2435	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2436	{
2437	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2438	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2439	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2440	}
2441
2442	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2443	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2444	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2445
2446	/*
2447	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2448	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2449	*/
2450	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2451	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2452	HMVMX_SET_TAGGED_TLB_FLUSHED();
2453	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2454	}
2455	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2456	{
2457	/*
2458	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2459	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2460	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2461	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2462	* mappings, see @bugref{6568}.
2463	*
2464	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2465	*/
2466	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2467	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2468	HMVMX_SET_TAGGED_TLB_FLUSHED();
2469	}
2470
2471	pVCpu->hm.s.fForceTLBFlush = false;
2472	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2473
2474	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2475	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2476	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2477	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2478	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2479	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2480	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2481	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2482	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2483
2484	/* Update VMCS with the VPID. */
2485	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2486	AssertRC(rc);
2487
2488	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2489	}
2490
2491
2492	/**
2493	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2494	*
2495	* @param pHostCpu The HM physical-CPU structure.
2496	* @param pVCpu The cross context virtual CPU structure.
2497	* @param pVmcsInfo The VMCS info. object.
2498	*
2499	* @remarks Called with interrupts disabled.
2500	*/
2501	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2502	{
2503	AssertPtr(pVCpu);
2504	AssertPtr(pHostCpu);
2505	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2506	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2507	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2508
2509	/*
2510	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2511	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2512	*/
2513	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2514	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2515	{
2516	pVCpu->hm.s.fForceTLBFlush = true;
2517	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2518	}
2519
2520	/* Check for explicit TLB flushes. */
2521	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2522	{
2523	pVCpu->hm.s.fForceTLBFlush = true;
2524	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2525	}
2526
2527	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2528	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2529
2530	if (pVCpu->hm.s.fForceTLBFlush)
2531	{
2532	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2533	pVCpu->hm.s.fForceTLBFlush = false;
2534	}
2535	}
2536
2537
2538	/**
2539	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2540	*
2541	* @param pHostCpu The HM physical-CPU structure.
2542	* @param pVCpu The cross context virtual CPU structure.
2543	*
2544	* @remarks Called with interrupts disabled.
2545	*/
2546	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2547	{
2548	AssertPtr(pVCpu);
2549	AssertPtr(pHostCpu);
2550	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2551	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2552	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2553
2554	/*
2555	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2556	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2557	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2558	* cannot reuse the current ASID anymore.
2559	*/
2560	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2561	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2562	{
2563	pVCpu->hm.s.fForceTLBFlush = true;
2564	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2565	}
2566
2567	/* Check for explicit TLB flushes. */
2568	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2569	{
2570	/*
2571	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2572	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2573	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2574	* include fExplicitFlush's too) - an obscure corner case.
2575	*/
2576	pVCpu->hm.s.fForceTLBFlush = true;
2577	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2578	}
2579
2580	PVM pVM = pVCpu->CTX_SUFF(pVM);
2581	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2582	if (pVCpu->hm.s.fForceTLBFlush)
2583	{
2584	++pHostCpu->uCurrentAsid;
2585	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2586	{
2587	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2588	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2589	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2590	}
2591
2592	pVCpu->hm.s.fForceTLBFlush = false;
2593	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2594	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2595	if (pHostCpu->fFlushAsidBeforeUse)
2596	{
2597	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2598	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2599	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2600	{
2601	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2602	pHostCpu->fFlushAsidBeforeUse = false;
2603	}
2604	else
2605	{
2606	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2607	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2608	}
2609	}
2610	}
2611
2612	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2613	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2614	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2615	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2616	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2617	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2618	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2619
2620	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2621	AssertRC(rc);
2622	}
2623
2624
2625	/**
2626	* Flushes the guest TLB entry based on CPU capabilities.
2627	*
2628	* @param pHostCpu The HM physical-CPU structure.
2629	* @param pVCpu The cross context virtual CPU structure.
2630	* @param pVmcsInfo The VMCS info. object.
2631	*
2632	* @remarks Called with interrupts disabled.
2633	*/
2634	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2635	{
2636	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2637	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2638	#endif
2639	PVM pVM = pVCpu->CTX_SUFF(pVM);
2640	switch (pVM->hm.s.vmx.enmTlbFlushType)
2641	{
2642	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2643	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2644	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2645	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2646	default:
2647	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2648	break;
2649	}
2650	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2651	}
2652
2653
2654	/**
2655	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2656	* TLB entries from the host TLB before VM-entry.
2657	*
2658	* @returns VBox status code.
2659	* @param pVM The cross context VM structure.
2660	*/
2661	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2662	{
2663	/*
2664	* Determine optimal flush type for nested paging.
2665	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2666	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2667	*/
2668	if (pVM->hm.s.fNestedPaging)
2669	{
2670	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2671	{
2672	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2673	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2674	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2675	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2676	else
2677	{
2678	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683
2684	/* Make sure the write-back cacheable memory type for EPT is supported. */
2685	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2686	{
2687	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2688	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2690	}
2691
2692	/* EPT requires a page-walk length of 4. */
2693	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2694	{
2695	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2696	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2697	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2698	}
2699	}
2700	else
2701	{
2702	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2703	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2704	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2705	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2706	}
2707	}
2708
2709	/*
2710	* Determine optimal flush type for VPID.
2711	*/
2712	if (pVM->hm.s.vmx.fVpid)
2713	{
2714	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2715	{
2716	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2717	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2718	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2720	else
2721	{
2722	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2723	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2724	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2725	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2726	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2727	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2728	pVM->hm.s.vmx.fVpid = false;
2729	}
2730	}
2731	else
2732	{
2733	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2734	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2735	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2736	pVM->hm.s.vmx.fVpid = false;
2737	}
2738	}
2739
2740	/*
2741	* Setup the handler for flushing tagged-TLBs.
2742	*/
2743	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2744	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2745	else if (pVM->hm.s.fNestedPaging)
2746	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2747	else if (pVM->hm.s.vmx.fVpid)
2748	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2749	else
2750	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2751	return VINF_SUCCESS;
2752	}
2753
2754
2755	/**
2756	* Sets up the virtual-APIC page address for the VMCS.
2757	*
2758	* @returns VBox status code.
2759	* @param pVCpu The cross context virtual CPU structure.
2760	* @param pVmcsInfo The VMCS info. object.
2761	*/
2762	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2763	{
2764	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2765	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2766	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2767	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2768	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2769	}
2770
2771
2772	/**
2773	* Sets up the MSR-bitmap address for the VMCS.
2774	*
2775	* @returns VBox status code.
2776	* @param pVCpu The cross context virtual CPU structure.
2777	* @param pVmcsInfo The VMCS info. object.
2778	*/
2779	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2780	{
2781	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2782	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2783	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2784	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2785	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2786	}
2787
2788
2789	/**
2790	* Sets up the APIC-access page address for the VMCS.
2791	*
2792	* @returns VBox status code.
2793	* @param pVCpu The cross context virtual CPU structure.
2794	*/
2795	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2796	{
2797	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2798	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2799	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2800	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2801	}
2802
2803
2804	/**
2805	* Sets up the VMCS link pointer for the VMCS.
2806	*
2807	* @returns VBox status code.
2808	* @param pVCpu The cross context virtual CPU structure.
2809	* @param pVmcsInfo The VMCS info. object.
2810	*/
2811	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2812	{
2813	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2814	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2815	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2816	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2817	}
2818
2819
2820	/**
2821	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2822	* in the VMCS.
2823	*
2824	* @returns VBox status code.
2825	* @param pVCpu The cross context virtual CPU structure.
2826	* @param pVmcsInfo The VMCS info. object.
2827	*/
2828	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2829	{
2830	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2831
2832	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2833	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2834	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2835
2836	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2837	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2838	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2839
2840	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2841	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2842	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2843
2844	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2845	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2846	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2847	AssertRCReturn(rc, rc);
2848	return VINF_SUCCESS;
2849	}
2850
2851
2852	/**
2853	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2854	*
2855	* @param pVCpu The cross context virtual CPU structure.
2856	* @param pVmcsInfo The VMCS info. object.
2857	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2858	*/
2859	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2860	{
2861	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2862
2863	/*
2864	* The guest can access the following MSRs (read, write) without causing
2865	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2866	*/
2867	PVM pVM = pVCpu->CTX_SUFF(pVM);
2868	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2869	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2871	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2872	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2873
2874	/*
2875	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2876	* associated with then. We never need to intercept access (writes need to be
2877	* executed without causing a VM-exit, reads will #GP fault anyway).
2878	*
2879	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2880	* read/write them. We swap the the guest/host MSR value using the
2881	* auto-load/store MSR area.
2882	*/
2883	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2884	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2885	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2886	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2887	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2888	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2889
2890	#if HC_ARCH_BITS == 64
2891	/*
2892	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2893	* required for 64-bit guests.
2894	*/
2895	if (pVM->hm.s.fAllow64BitGuests)
2896	{
2897	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2898	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2899	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2900	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2901	}
2902	#endif
2903
2904	/*
2905	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2906	*/
2907	#ifdef VBOX_STRICT
2908	Assert(pVmcsInfo->pvMsrBitmap);
2909	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2910	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2911	#endif
2912	}
2913
2914
2915	/**
2916	* Sets up pin-based VM-execution controls in the VMCS.
2917	*
2918	* @returns VBox status code.
2919	* @param pVCpu The cross context virtual CPU structure.
2920	* @param pVmcsInfo The VMCS info. object.
2921	*/
2922	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2923	{
2924	PVM pVM = pVCpu->CTX_SUFF(pVM);
2925	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2926	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2927
2928	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2929	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2930
2931	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2932	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2933
2934	/* Enable the VMX-preemption timer. */
2935	if (pVM->hm.s.vmx.fUsePreemptTimer)
2936	{
2937	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2938	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2939	}
2940
2941	#if 0
2942	/* Enable posted-interrupt processing. */
2943	if (pVM->hm.s.fPostedIntrs)
2944	{
2945	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2946	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2947	fVal \|= VMX_PIN_CTL_POSTED_INT;
2948	}
2949	#endif
2950
2951	if ((fVal & fZap) != fVal)
2952	{
2953	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2954	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2955	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2956	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2957	}
2958
2959	/* Commit it to the VMCS and update our cache. */
2960	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2961	AssertRCReturn(rc, rc);
2962	pVmcsInfo->u32PinCtls = fVal;
2963
2964	return VINF_SUCCESS;
2965	}
2966
2967
2968	/**
2969	* Sets up secondary processor-based VM-execution controls in the VMCS.
2970	*
2971	* @returns VBox status code.
2972	* @param pVCpu The cross context virtual CPU structure.
2973	* @param pVmcsInfo The VMCS info. object.
2974	*/
2975	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2976	{
2977	PVM pVM = pVCpu->CTX_SUFF(pVM);
2978	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2979	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2980
2981	/* WBINVD causes a VM-exit. */
2982	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2983	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2984
2985	/* Enable EPT (aka nested-paging). */
2986	if (pVM->hm.s.fNestedPaging)
2987	fVal \|= VMX_PROC_CTLS2_EPT;
2988
2989	/* Enable the INVPCID instruction if supported by the hardware and we expose
2990	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2991	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2992	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2993	fVal \|= VMX_PROC_CTLS2_INVPCID;
2994
2995	/* Enable VPID. */
2996	if (pVM->hm.s.vmx.fVpid)
2997	fVal \|= VMX_PROC_CTLS2_VPID;
2998
2999	/* Enable unrestricted guest execution. */
3000	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3001	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3002
3003	#if 0
3004	if (pVM->hm.s.fVirtApicRegs)
3005	{
3006	/* Enable APIC-register virtualization. */
3007	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3008	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3009
3010	/* Enable virtual-interrupt delivery. */
3011	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3012	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3013	}
3014	#endif
3015
3016	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3017	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3018	* done dynamically. */
3019	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3020	{
3021	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3022	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3023	AssertRCReturn(rc, rc);
3024	}
3025
3026	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3027	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3028	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3029	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3030	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3031
3032	/* Enable Pause-Loop exiting. */
3033	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3034	&& pVM->hm.s.vmx.cPleGapTicks
3035	&& pVM->hm.s.vmx.cPleWindowTicks)
3036	{
3037	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3038
3039	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3040	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3041	AssertRCReturn(rc, rc);
3042	}
3043
3044	if ((fVal & fZap) != fVal)
3045	{
3046	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3047	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3048	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3049	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3050	}
3051
3052	/* Commit it to the VMCS and update our cache. */
3053	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3054	AssertRCReturn(rc, rc);
3055	pVmcsInfo->u32ProcCtls2 = fVal;
3056
3057	return VINF_SUCCESS;
3058	}
3059
3060
3061	/**
3062	* Sets up processor-based VM-execution controls in the VMCS.
3063	*
3064	* @returns VBox status code.
3065	* @param pVCpu The cross context virtual CPU structure.
3066	* @param pVmcsInfo The VMCS info. object.
3067	*/
3068	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3069	{
3070	PVM pVM = pVCpu->CTX_SUFF(pVM);
3071
3072	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3073	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3074
3075	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3076	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3077	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3078	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3079	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3080	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3081	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3082
3083	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3084	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3085	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3086	{
3087	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3088	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3089	}
3090
3091	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3092	if (!pVM->hm.s.fNestedPaging)
3093	{
3094	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3095	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3096	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3097	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3098	}
3099
3100	/* Use TPR shadowing if supported by the CPU. */
3101	if ( PDMHasApic(pVM)
3102	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3103	{
3104	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3105	/* CR8 writes cause a VM-exit based on TPR threshold. */
3106	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3107	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3108	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3109	AssertRCReturn(rc, rc);
3110	}
3111	else
3112	{
3113	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3114	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3115	if (pVM->hm.s.fAllow64BitGuests)
3116	{
3117	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3118	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3119	}
3120	}
3121
3122	/* Use MSR-bitmaps if supported by the CPU. */
3123	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3124	{
3125	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3126	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3127	AssertRCReturn(rc, rc);
3128	}
3129
3130	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3131	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3132	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3133
3134	if ((fVal & fZap) != fVal)
3135	{
3136	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3137	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3138	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3139	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3140	}
3141
3142	/* Commit it to the VMCS and update our cache. */
3143	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3144	AssertRCReturn(rc, rc);
3145	pVmcsInfo->u32ProcCtls = fVal;
3146
3147	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3148	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3149	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3150
3151	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3152	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3153	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3154
3155	/* Sanity check, should not really happen. */
3156	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3157	{ /* likely */ }
3158	else
3159	{
3160	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3161	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3162	}
3163
3164	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3165	return VINF_SUCCESS;
3166	}
3167
3168
3169	/**
3170	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3171	* Processor-based VM-execution) control fields in the VMCS.
3172	*
3173	* @returns VBox status code.
3174	* @param pVCpu The cross context virtual CPU structure.
3175	* @param pVmcsInfo The VMCS info. object.
3176	*/
3177	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3178	{
3179	/* Set the auto-load/store MSR area addresses in the VMCS. */
3180	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3181	if (RT_SUCCESS(rc))
3182	{
3183	/* Set the VMCS link pointer in the VMCS. */
3184	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3185	if (RT_SUCCESS(rc))
3186	{
3187	/* Set the CR0/CR4 guest/host mask. */
3188	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3189	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3190	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3191	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3192	if (RT_SUCCESS(rc))
3193	{
3194	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3195	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3196	return VINF_SUCCESS;
3197	}
3198	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3199	}
3200	else
3201	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3202	}
3203	else
3204	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3205	return rc;
3206	}
3207
3208
3209	/**
3210	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3211	*
3212	* We shall setup those exception intercepts that don't change during the
3213	* lifetime of the VM here. The rest are done dynamically while loading the
3214	* guest state.
3215	*
3216	* @returns VBox status code.
3217	* @param pVCpu The cross context virtual CPU structure.
3218	* @param pVmcsInfo The VMCS info. object.
3219	*/
3220	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3221	{
3222	/*
3223	* The following exceptions are always intercepted:
3224	*
3225	* #AC - To prevent the guest from hanging the CPU.
3226	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3227	* recursive #DBs can cause a CPU hang.
3228	* #PF - To sync our shadow page tables when nested-paging is not used.
3229	*/
3230	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3231	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3232	\| RT_BIT(X86_XCPT_DB)
3233	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3234
3235	/* Commit it to the VMCS. */
3236	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3237	AssertRCReturn(rc, rc);
3238
3239	/* Update our cache of the exception bitmap. */
3240	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3241	return VINF_SUCCESS;
3242	}
3243
3244
3245	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3246	/**
3247	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3248	*
3249	* @returns VBox status code.
3250	* @param pVCpu The cross context virtual CPU structure.
3251	* @param pVmcsInfo The VMCS info. object.
3252	*/
3253	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3254	{
3255	PVM pVM = pVCpu->CTX_SUFF(pVM);
3256	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3257	if (RT_SUCCESS(rc))
3258	{
3259	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3260	if (RT_SUCCESS(rc))
3261	{
3262	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3263	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3264	if (RT_SUCCESS(rc))
3265	{
3266	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3267	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3268	if (RT_SUCCESS(rc))
3269	return VINF_SUCCESS;
3270
3271	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3272	}
3273	else
3274	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3275	}
3276	else
3277	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3278	}
3279	else
3280	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3281
3282	return rc;
3283	}
3284	#endif
3285
3286
3287	/**
3288	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3289	* VMX.
3290	*
3291	* @returns VBox status code.
3292	* @param pVCpu The cross context virtual CPU structure.
3293	* @param pVmcsInfo The VMCS info. object.
3294	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3295	*/
3296	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3297	{
3298	Assert(pVmcsInfo);
3299	Assert(pVmcsInfo->pvVmcs);
3300	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3301
3302	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3303	PVM pVM = pVCpu->CTX_SUFF(pVM);
3304	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3305	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3306
3307	LogFlowFunc(("\n"));
3308
3309	/*
3310	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3311	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3312	*/
3313	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3314	if (RT_SUCCESS(rc))
3315	{
3316	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3317	if (RT_SUCCESS(rc))
3318	{
3319	if (!fIsNstGstVmcs)
3320	{
3321	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3322	if (RT_SUCCESS(rc))
3323	{
3324	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3325	if (RT_SUCCESS(rc))
3326	{
3327	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3328	if (RT_SUCCESS(rc))
3329	{
3330	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3331	if (RT_SUCCESS(rc))
3332	{ /* likely */ }
3333	else
3334	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3335	}
3336	else
3337	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3338	}
3339	else
3340	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3341	}
3342	else
3343	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3344	}
3345	else
3346	{
3347	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3348	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3349	if (RT_SUCCESS(rc))
3350	{ /* likely */ }
3351	else
3352	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3353	#else
3354	AssertFailed();
3355	#endif
3356	}
3357	}
3358	else
3359	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3360	}
3361	else
3362	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3363
3364	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3365	if (RT_SUCCESS(rc))
3366	{
3367	rc = hmR0VmxClearVmcs(pVmcsInfo);
3368	if (RT_SUCCESS(rc))
3369	{ /* likely */ }
3370	else
3371	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3372	}
3373
3374	/*
3375	* Update the last-error record both for failures and success, so we
3376	* can propagate the status code back to ring-3 for diagnostics.
3377	*/
3378	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3379	NOREF(pszVmcs);
3380	return rc;
3381	}
3382
3383
3384	/**
3385	* Does global VT-x initialization (called during module initialization).
3386	*
3387	* @returns VBox status code.
3388	*/
3389	VMMR0DECL(int) VMXR0GlobalInit(void)
3390	{
3391	#ifdef HMVMX_USE_FUNCTION_TABLE
3392	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3393	# ifdef VBOX_STRICT
3394	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3395	Assert(g_apfnVMExitHandlers[i]);
3396	# endif
3397	#endif
3398	return VINF_SUCCESS;
3399	}
3400
3401
3402	/**
3403	* Does global VT-x termination (called during module termination).
3404	*/
3405	VMMR0DECL(void) VMXR0GlobalTerm()
3406	{
3407	/* Nothing to do currently. */
3408	}
3409
3410
3411	/**
3412	* Sets up and activates VT-x on the current CPU.
3413	*
3414	* @returns VBox status code.
3415	* @param pHostCpu The HM physical-CPU structure.
3416	* @param pVM The cross context VM structure. Can be
3417	* NULL after a host resume operation.
3418	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3419	* fEnabledByHost is @c true).
3420	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3421	* @a fEnabledByHost is @c true).
3422	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3423	* enable VT-x on the host.
3424	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3425	*/
3426	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3427	PCSUPHWVIRTMSRS pHwvirtMsrs)
3428	{
3429	Assert(pHostCpu);
3430	Assert(pHwvirtMsrs);
3431	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3432
3433	/* Enable VT-x if it's not already enabled by the host. */
3434	if (!fEnabledByHost)
3435	{
3436	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3437	if (RT_FAILURE(rc))
3438	return rc;
3439	}
3440
3441	/*
3442	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3443	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3444	* invalidated when flushing by VPID.
3445	*/
3446	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3447	{
3448	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3449	pHostCpu->fFlushAsidBeforeUse = false;
3450	}
3451	else
3452	pHostCpu->fFlushAsidBeforeUse = true;
3453
3454	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3455	++pHostCpu->cTlbFlushes;
3456
3457	return VINF_SUCCESS;
3458	}
3459
3460
3461	/**
3462	* Deactivates VT-x on the current CPU.
3463	*
3464	* @returns VBox status code.
3465	* @param pvCpuPage Pointer to the VMXON region.
3466	* @param HCPhysCpuPage Physical address of the VMXON region.
3467	*
3468	* @remarks This function should never be called when SUPR0EnableVTx() or
3469	* similar was used to enable VT-x on the host.
3470	*/
3471	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3472	{
3473	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3474
3475	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3476	return hmR0VmxLeaveRootMode();
3477	}
3478
3479
3480	/**
3481	* Does per-VM VT-x initialization.
3482	*
3483	* @returns VBox status code.
3484	* @param pVM The cross context VM structure.
3485	*/
3486	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3487	{
3488	LogFlowFunc(("pVM=%p\n", pVM));
3489
3490	int rc = hmR0VmxStructsAlloc(pVM);
3491	if (RT_FAILURE(rc))
3492	{
3493	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3494	return rc;
3495	}
3496
3497	return VINF_SUCCESS;
3498	}
3499
3500
3501	/**
3502	* Does per-VM VT-x termination.
3503	*
3504	* @returns VBox status code.
3505	* @param pVM The cross context VM structure.
3506	*/
3507	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3508	{
3509	LogFlowFunc(("pVM=%p\n", pVM));
3510
3511	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3512	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3513	{
3514	Assert(pVM->hm.s.vmx.pvScratch);
3515	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3516	}
3517	#endif
3518	hmR0VmxStructsFree(pVM);
3519	return VINF_SUCCESS;
3520	}
3521
3522
3523	/**
3524	* Sets up the VM for execution using hardware-assisted VMX.
3525	* This function is only called once per-VM during initialization.
3526	*
3527	* @returns VBox status code.
3528	* @param pVM The cross context VM structure.
3529	*/
3530	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3531	{
3532	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3533	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3534
3535	LogFlowFunc(("pVM=%p\n", pVM));
3536
3537	/*
3538	* At least verify if VMX is enabled, since we can't check if we're in
3539	* VMX root mode or not without causing a #GP.
3540	*/
3541	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3542	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3543	{ /* likely */ }
3544	else
3545	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3546
3547	/*
3548	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3549	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3550	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3551	*/
3552	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3553	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3554	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3555	{
3556	LogRelFunc(("Invalid real-on-v86 state.\n"));
3557	return VERR_INTERNAL_ERROR;
3558	}
3559
3560	/* Initialize these always, see hmR3InitFinalizeR0().*/
3561	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3562	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3563
3564	/* Setup the tagged-TLB flush handlers. */
3565	int rc = hmR0VmxSetupTaggedTlb(pVM);
3566	if (RT_FAILURE(rc))
3567	{
3568	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3569	return rc;
3570	}
3571
3572	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3573	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3574	#if HC_ARCH_BITS == 64
3575	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3576	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3577	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3578	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3579	#endif
3580
3581	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3582	{
3583	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3584	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3585
3586	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3587	if (RT_SUCCESS(rc))
3588	{
3589	#if HC_ARCH_BITS == 32
3590	hmR0VmxInitVmcsReadCache(pVCpu);
3591	#endif
3592	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3593	if (pVM->cpum.ro.GuestFeatures.fVmx)
3594	{
3595	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3596	if (RT_SUCCESS(rc))
3597	{ /* likely */ }
3598	else
3599	{
3600	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3601	return rc;
3602	}
3603	}
3604	#endif
3605	}
3606	else
3607	{
3608	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3609	return rc;
3610	}
3611	}
3612
3613	return VINF_SUCCESS;
3614	}
3615
3616
3617	#if HC_ARCH_BITS == 32
3618	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3619	/**
3620	* Check if guest state allows safe use of 32-bit switcher again.
3621	*
3622	* Segment bases and protected mode structures must be 32-bit addressable
3623	* because the 32-bit switcher will ignore high dword when writing these VMCS
3624	* fields. See @bugref{8432} for details.
3625	*
3626	* @returns true if safe, false if must continue to use the 64-bit switcher.
3627	* @param pCtx Pointer to the guest-CPU context.
3628	*
3629	* @remarks No-long-jump zone!!!
3630	*/
3631	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3632	{
3633	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3634	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3635	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3636	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3637	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3638	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3639	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3640	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3641	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3642	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3643
3644	/* All good, bases are 32-bit. */
3645	return true;
3646	}
3647	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3648
3649	# ifdef VBOX_STRICT
3650	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3651	{
3652	switch (idxField)
3653	{
3654	case VMX_VMCS_GUEST_RIP:
3655	case VMX_VMCS_GUEST_RSP:
3656	case VMX_VMCS_GUEST_SYSENTER_EIP:
3657	case VMX_VMCS_GUEST_SYSENTER_ESP:
3658	case VMX_VMCS_GUEST_GDTR_BASE:
3659	case VMX_VMCS_GUEST_IDTR_BASE:
3660	case VMX_VMCS_GUEST_CS_BASE:
3661	case VMX_VMCS_GUEST_DS_BASE:
3662	case VMX_VMCS_GUEST_ES_BASE:
3663	case VMX_VMCS_GUEST_FS_BASE:
3664	case VMX_VMCS_GUEST_GS_BASE:
3665	case VMX_VMCS_GUEST_SS_BASE:
3666	case VMX_VMCS_GUEST_LDTR_BASE:
3667	case VMX_VMCS_GUEST_TR_BASE:
3668	case VMX_VMCS_GUEST_CR3:
3669	return true;
3670	}
3671	return false;
3672	}
3673
3674	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3675	{
3676	switch (idxField)
3677	{
3678	/* Read-only fields. */
3679	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3680	return true;
3681	}
3682	/* Remaining readable fields should also be writable. */
3683	return hmR0VmxIsValidWriteField(idxField);
3684	}
3685	# endif /* VBOX_STRICT */
3686
3687
3688	/**
3689	* Executes the specified handler in 64-bit mode.
3690	*
3691	* @returns VBox status code (no informational status codes).
3692	* @param pVCpu The cross context virtual CPU structure.
3693	* @param enmOp The operation to perform.
3694	* @param cParams Number of parameters.
3695	* @param paParam Array of 32-bit parameters.
3696	*/
3697	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3698	{
3699	PVM pVM = pVCpu->CTX_SUFF(pVM);
3700	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3701	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3702	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3703	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3704
3705	#ifdef VBOX_STRICT
3706	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3707	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3708
3709	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3710	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3711	#endif
3712
3713	/* Disable interrupts. */
3714	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3715
3716	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3717	RTCPUID idHostCpu = RTMpCpuId();
3718	CPUMR0SetLApic(pVCpu, idHostCpu);
3719	#endif
3720
3721	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3722
3723	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3724	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3725
3726	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3727	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3728	hmR0VmxClearVmcs(pVmcsInfo);
3729
3730	/* Leave VMX root mode and disable VMX. */
3731	VMXDisable();
3732	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3733
3734	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3735	CPUMSetHyperEIP(pVCpu, enmOp);
3736	for (int i = (int)cParams - 1; i >= 0; i--)
3737	CPUMPushHyper(pVCpu, paParam[i]);
3738
3739	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3740
3741	/* Call the switcher. */
3742	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3743	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3744
3745	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3746	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3747
3748	/* Re-enter VMX root mode. */
3749	int rc2 = VMXEnable(HCPhysCpuPage);
3750	if (RT_FAILURE(rc2))
3751	{
3752	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3753	ASMSetFlags(fOldEFlags);
3754	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3755	return rc2;
3756	}
3757
3758	/* Restore the VMCS as the current VMCS. */
3759	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3760	AssertRC(rc2);
3761	Assert(!(ASMGetFlags() & X86_EFL_IF));
3762	ASMSetFlags(fOldEFlags);
3763	return rc;
3764	}
3765
3766
3767	/**
3768	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3769	* supporting 64-bit guests.
3770	*
3771	* @returns VBox status code.
3772	* @param fResume Whether to VMLAUNCH or VMRESUME.
3773	* @param pCtx Pointer to the guest-CPU context.
3774	* @param pCache Pointer to the VMCS batch cache.
3775	* @param pVM The cross context VM structure.
3776	* @param pVCpu The cross context virtual CPU structure.
3777	*/
3778	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3779	{
3780	NOREF(fResume);
3781
3782	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3783	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3784	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3785
3786	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3787	pCache->uPos = 1;
3788	pCache->interPD = PGMGetInterPaeCR3(pVM);
3789	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3790	#endif
3791
3792	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3793	pCache->TestIn.HCPhysCpuPage = 0;
3794	pCache->TestIn.HCPhysVmcs = 0;
3795	pCache->TestIn.pCache = 0;
3796	pCache->TestOut.HCPhysVmcs = 0;
3797	pCache->TestOut.pCache = 0;
3798	pCache->TestOut.pCtx = 0;
3799	pCache->TestOut.eflags = 0;
3800	#else
3801	NOREF(pCache);
3802	#endif
3803
3804	uint32_t aParam[10];
3805	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3806	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3807	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3808	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3809	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3810	aParam[5] = 0;
3811	aParam[6] = VM_RC_ADDR(pVM, pVM);
3812	aParam[7] = 0;
3813	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3814	aParam[9] = 0;
3815
3816	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3817	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3818	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3819	#endif
3820	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3821
3822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3823	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3824	Assert(pCtx->dr[4] == 10);
3825	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3826	#endif
3827
3828	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3829	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3830	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3831	pVmcsInfo->HCPhysVmcs));
3832	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3833	pCache->TestOut.HCPhysVmcs));
3834	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3835	pCache->TestOut.pCache));
3836	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3837	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3838	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3839	pCache->TestOut.pCtx));
3840	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3841	#endif
3842	NOREF(pCtx);
3843	return rc;
3844	}
3845	#endif
3846
3847
3848	/**
3849	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3850	* the VMCS.
3851	*
3852	* @returns VBox status code.
3853	*/
3854	static int hmR0VmxExportHostControlRegs(void)
3855	{
3856	RTCCUINTREG uReg = ASMGetCR0();
3857	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3858	AssertRCReturn(rc, rc);
3859
3860	uReg = ASMGetCR3();
3861	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3862	AssertRCReturn(rc, rc);
3863
3864	uReg = ASMGetCR4();
3865	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3866	AssertRCReturn(rc, rc);
3867	return rc;
3868	}
3869
3870
3871	/**
3872	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3873	* the host-state area in the VMCS.
3874	*
3875	* @returns VBox status code.
3876	* @param pVCpu The cross context virtual CPU structure.
3877	*/
3878	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3879	{
3880	#if HC_ARCH_BITS == 64
3881	/**
3882	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3883	* requirements. See hmR0VmxExportHostSegmentRegs().
3884	*/
3885	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3886	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3887	{ \
3888	bool fValidSelector = true; \
3889	if ((selValue) & X86_SEL_LDT) \
3890	{ \
3891	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3892	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3893	} \
3894	if (fValidSelector) \
3895	{ \
3896	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3897	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3898	} \
3899	(selValue) = 0; \
3900	}
3901
3902	/*
3903	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3904	* will be messed up. We should -not- save the messed up state without restoring
3905	* the original host-state, see @bugref{7240}.
3906	*
3907	* This apparently can happen (most likely the FPU changes), deal with it rather than
3908	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3909	*/
3910	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3911	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3912	{
3913	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3914	pVCpu->idCpu));
3915	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3916	}
3917	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3918	#else
3919	RT_NOREF(pVCpu);
3920	#endif
3921
3922	/*
3923	* Host DS, ES, FS and GS segment registers.
3924	*/
3925	#if HC_ARCH_BITS == 64
3926	RTSEL uSelDS = ASMGetDS();
3927	RTSEL uSelES = ASMGetES();
3928	RTSEL uSelFS = ASMGetFS();
3929	RTSEL uSelGS = ASMGetGS();
3930	#else
3931	RTSEL uSelDS = 0;
3932	RTSEL uSelES = 0;
3933	RTSEL uSelFS = 0;
3934	RTSEL uSelGS = 0;
3935	#endif
3936
3937	/*
3938	* Host CS and SS segment registers.
3939	*/
3940	RTSEL uSelCS = ASMGetCS();
3941	RTSEL uSelSS = ASMGetSS();
3942
3943	/*
3944	* Host TR segment register.
3945	*/
3946	RTSEL uSelTR = ASMGetTR();
3947
3948	#if HC_ARCH_BITS == 64
3949	/*
3950	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3951	* gain VM-entry and restore them before we get preempted.
3952	*
3953	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3954	*/
3955	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3956	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3957	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3958	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3959	# undef VMXLOCAL_ADJUST_HOST_SEG
3960	#endif
3961
3962	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3963	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3964	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3965	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3966	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3967	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3968	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3969	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3970	Assert(uSelCS);
3971	Assert(uSelTR);
3972
3973	/* Write these host selector fields into the host-state area in the VMCS. */
3974	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3975	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3976	#if HC_ARCH_BITS == 64
3977	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3979	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3980	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3981	#else
3982	NOREF(uSelDS);
3983	NOREF(uSelES);
3984	NOREF(uSelFS);
3985	NOREF(uSelGS);
3986	#endif
3987	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3988	AssertRCReturn(rc, rc);
3989
3990	/*
3991	* Host GDTR and IDTR.
3992	*/
3993	RTGDTR Gdtr;
3994	RTIDTR Idtr;
3995	RT_ZERO(Gdtr);
3996	RT_ZERO(Idtr);
3997	ASMGetGDTR(&Gdtr);
3998	ASMGetIDTR(&Idtr);
3999	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4000	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4001	AssertRCReturn(rc, rc);
4002
4003	#if HC_ARCH_BITS == 64
4004	/*
4005	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4006	* them to the maximum limit (0xffff) on every VM-exit.
4007	*/
4008	if (Gdtr.cbGdt != 0xffff)
4009	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4010
4011	/*
4012	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4013	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4014	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4015	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4016	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4017	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4018	* at 0xffff on hosts where we are sure it won't cause trouble.
4019	*/
4020	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4021	if (Idtr.cbIdt < 0x0fff)
4022	# else
4023	if (Idtr.cbIdt != 0xffff)
4024	# endif
4025	{
4026	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4027	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4028	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4029	}
4030	#endif
4031
4032	/*
4033	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4034	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4035	* RPL should be too in most cases.
4036	*/
4037	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4038	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4039
4040	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4041	#if HC_ARCH_BITS == 64
4042	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4043
4044	/*
4045	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4046	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4047	* restoration if the host has something else. Task switching is not supported in 64-bit
4048	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4049	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4050	*
4051	* [1] See Intel spec. 3.5 "System Descriptor Types".
4052	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4053	*/
4054	PVM pVM = pVCpu->CTX_SUFF(pVM);
4055	Assert(pDesc->System.u4Type == 11);
4056	if ( pDesc->System.u16LimitLow != 0x67
4057	\|\| pDesc->System.u4LimitHigh)
4058	{
4059	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4060	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4061	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4062	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4063	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4064	}
4065
4066	/*
4067	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4068	*/
4069	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4070	{
4071	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4072	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4073	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4074	{
4075	/* The GDT is read-only but the writable GDT is available. */
4076	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4077	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4078	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4079	AssertRCReturn(rc, rc);
4080	}
4081	}
4082	#else
4083	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4084	#endif
4085	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4086	AssertRCReturn(rc, rc);
4087
4088	/*
4089	* Host FS base and GS base.
4090	*/
4091	#if HC_ARCH_BITS == 64
4092	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4093	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4094	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4095	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4096	AssertRCReturn(rc, rc);
4097
4098	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4099	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4100	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4101	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4102	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4103	#endif
4104	return VINF_SUCCESS;
4105	}
4106
4107
4108	/**
4109	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4110	* host-state area of the VMCS.
4111	*
4112	* These MSRs will be automatically restored on the host after every successful
4113	* VM-exit.
4114	*
4115	* @returns VBox status code.
4116	* @param pVCpu The cross context virtual CPU structure.
4117	*
4118	* @remarks No-long-jump zone!!!
4119	*/
4120	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4121	{
4122	AssertPtr(pVCpu);
4123
4124	/*
4125	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4126	* rather than swapping them on every VM-entry.
4127	*/
4128	hmR0VmxLazySaveHostMsrs(pVCpu);
4129
4130	/*
4131	* Host Sysenter MSRs.
4132	*/
4133	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4134	#if HC_ARCH_BITS == 32
4135	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4136	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4137	#else
4138	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4139	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4140	#endif
4141	AssertRCReturn(rc, rc);
4142
4143	/*
4144	* Host EFER MSR.
4145	*
4146	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4147	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4148	*/
4149	PVM pVM = pVCpu->CTX_SUFF(pVM);
4150	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4151	{
4152	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4153	AssertRCReturn(rc, rc);
4154	}
4155
4156	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4157	* hmR0VmxExportGuestEntryExitCtls(). */
4158
4159	return VINF_SUCCESS;
4160	}
4161
4162
4163	/**
4164	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4165	*
4166	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4167	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4168	*
4169	* @returns true if we need to load guest EFER, false otherwise.
4170	* @param pVCpu The cross context virtual CPU structure.
4171	*
4172	* @remarks Requires EFER, CR4.
4173	* @remarks No-long-jump zone!!!
4174	*/
4175	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4176	{
4177	#ifdef HMVMX_ALWAYS_SWAP_EFER
4178	RT_NOREF(pVCpu);
4179	return true;
4180	#else
4181	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4182	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4183	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4184	if (CPUMIsGuestInLongModeEx(pCtx))
4185	return false;
4186	#endif
4187
4188	PVM pVM = pVCpu->CTX_SUFF(pVM);
4189	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4190	uint64_t const u64GuestEfer = pCtx->msrEFER;
4191
4192	/*
4193	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4194	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4195	*/
4196	if ( CPUMIsGuestInLongModeEx(pCtx)
4197	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4198	return true;
4199
4200	/*
4201	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4202	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4203	*
4204	* See Intel spec. 4.5 "IA-32e Paging".
4205	* See Intel spec. 4.1.1 "Three Paging Modes".
4206	*
4207	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4208	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4209	*/
4210	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4211	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4212	if ( (pCtx->cr4 & X86_CR4_PAE)
4213	&& (pCtx->cr0 & X86_CR0_PG)
4214	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4215	{
4216	/* Assert that host is NX capable. */
4217	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4218	return true;
4219	}
4220
4221	return false;
4222	#endif
4223	}
4224
4225	/**
4226	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4227	* VMCS.
4228	*
4229	* This is typically required when the guest changes paging mode.
4230	*
4231	* @returns VBox status code.
4232	* @param pVCpu The cross context virtual CPU structure.
4233	* @param pVmxTransient The VMX-transient structure.
4234	*
4235	* @remarks Requires EFER.
4236	* @remarks No-long-jump zone!!!
4237	*/
4238	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4239	{
4240	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4241	{
4242	PVM pVM = pVCpu->CTX_SUFF(pVM);
4243	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4244
4245	/*
4246	* VM-entry controls.
4247	*/
4248	{
4249	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4250	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4251
4252	/*
4253	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4254	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4255	*
4256	* For nested-guests, this is a mandatory VM-entry control. It's also
4257	* required because we do not want to leak host bits to the nested-guest.
4258	*/
4259	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4260
4261	/*
4262	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4263	*
4264	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4265	* required to get the nested-guest working with hardware-assisted VMX execution.
4266	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4267	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4268	* here rather than while merging the guest VMCS controls.
4269	*/
4270	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4271	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4272	else
4273	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4274
4275	/*
4276	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4277	*
4278	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4279	* regardless of whether the nested-guest VMCS specifies it because we are free to
4280	* load whatever MSRs we require and we do not need to modify the guest visible copy
4281	* of the VM-entry MSR load area.
4282	*/
4283	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4284	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4285	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4286	else
4287	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4288
4289	/*
4290	* The following should -not- be set (since we're not in SMM mode):
4291	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4292	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4293	*/
4294
4295	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4296	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4297
4298	if ((fVal & fZap) == fVal)
4299	{ /* likely */ }
4300	else
4301	{
4302	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4303	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4304	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4305	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4306	}
4307
4308	/* Commit it to the VMCS. */
4309	if (pVmcsInfo->u32EntryCtls != fVal)
4310	{
4311	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4312	AssertRCReturn(rc, rc);
4313	pVmcsInfo->u32EntryCtls = fVal;
4314	}
4315	}
4316
4317	/*
4318	* VM-exit controls.
4319	*/
4320	{
4321	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4322	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4323
4324	/*
4325	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4326	* supported the 1-setting of this bit.
4327	*
4328	* For nested-guests, we set the "save debug controls" as the converse
4329	* "load debug controls" is mandatory for nested-guests anyway.
4330	*/
4331	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4332
4333	/*
4334	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4335	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4336	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4337	* hmR0VmxExportHostMsrs().
4338	*
4339	* For nested-guests, we always set this bit as we do not support 32-bit
4340	* hosts.
4341	*/
4342	#if HC_ARCH_BITS == 64
4343	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4344	#else
4345	Assert(!pVmxTransient->fIsNestedGuest);
4346	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4347	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4348	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4349	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4350	{
4351	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4352	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4353	}
4354	else
4355	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4356	#endif
4357
4358	/*
4359	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4360	*
4361	* For nested-guests, we should use the "save IA32_EFER" control if we also
4362	* used the "load IA32_EFER" control while exporting VM-entry controls.
4363	*/
4364	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4365	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4366	{
4367	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4368	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4369	}
4370
4371	/*
4372	* Enable saving of the VMX-preemption timer value on VM-exit.
4373	* For nested-guests, currently not exposed/used.
4374	*/
4375	if ( pVM->hm.s.vmx.fUsePreemptTimer
4376	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4377	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4378
4379	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4380	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4381
4382	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4383	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4384	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4385
4386	if ((fVal & fZap) == fVal)
4387	{ /* likely */ }
4388	else
4389	{
4390	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4391	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4392	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4393	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4394	}
4395
4396	/* Commit it to the VMCS. */
4397	if (pVmcsInfo->u32ExitCtls != fVal)
4398	{
4399	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4400	AssertRCReturn(rc, rc);
4401	pVmcsInfo->u32ExitCtls = fVal;
4402	}
4403	}
4404
4405	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4406	}
4407	return VINF_SUCCESS;
4408	}
4409
4410
4411	/**
4412	* Sets the TPR threshold in the VMCS.
4413	*
4414	* @returns VBox status code.
4415	* @param pVCpu The cross context virtual CPU structure.
4416	* @param pVmcsInfo The VMCS info. object.
4417	* @param u32TprThreshold The TPR threshold (task-priority class only).
4418	*/
4419	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4420	{
4421	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4422	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4423	RT_NOREF2(pVCpu, pVmcsInfo);
4424	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4425	}
4426
4427
4428	/**
4429	* Exports the guest APIC TPR state into the VMCS.
4430	*
4431	* @returns VBox status code.
4432	* @param pVCpu The cross context virtual CPU structure.
4433	* @param pVmxTransient The VMX-transient structure.
4434	*
4435	* @remarks No-long-jump zone!!!
4436	*/
4437	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4438	{
4439	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4440	{
4441	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4442
4443	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4444	if (!pVmxTransient->fIsNestedGuest)
4445	{
4446	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4447	&& APICIsEnabled(pVCpu))
4448	{
4449	/*
4450	* Setup TPR shadowing.
4451	*/
4452	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4453	{
4454	bool fPendingIntr = false;
4455	uint8_t u8Tpr = 0;
4456	uint8_t u8PendingIntr = 0;
4457	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4458	AssertRCReturn(rc, rc);
4459
4460	/*
4461	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4462	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4463	* priority of the pending interrupt so we can deliver the interrupt. If there
4464	* are no interrupts pending, set threshold to 0 to not cause any
4465	* TPR-below-threshold VM-exits.
4466	*/
4467	Assert(pVmcsInfo->pbVirtApic);
4468	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4469	uint32_t u32TprThreshold = 0;
4470	if (fPendingIntr)
4471	{
4472	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4473	(which is the Task-Priority Class). */
4474	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4475	const uint8_t u8TprPriority = u8Tpr >> 4;
4476	if (u8PendingPriority <= u8TprPriority)
4477	u32TprThreshold = u8PendingPriority;
4478	}
4479
4480	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4481	AssertRCReturn(rc, rc);
4482	}
4483	}
4484	}
4485	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4486	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4487	}
4488	return VINF_SUCCESS;
4489	}
4490
4491
4492	/**
4493	* Gets the guest interruptibility-state.
4494	*
4495	* @returns Guest's interruptibility-state.
4496	* @param pVCpu The cross context virtual CPU structure.
4497	* @param pVmcsInfo The VMCS info. object.
4498	*
4499	* @remarks No-long-jump zone!!!
4500	*/
4501	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4502	{
4503	/*
4504	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4505	*/
4506	uint32_t fIntrState = 0;
4507	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4508	{
4509	/* If inhibition is active, RIP and RFLAGS should've been updated
4510	(i.e. read previously from the VMCS or from ring-3). */
4511	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4512	#ifdef VBOX_STRICT
4513	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4514	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4515	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4516	#endif
4517	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4518	{
4519	if (pCtx->eflags.Bits.u1IF)
4520	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4521	else
4522	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4523	}
4524	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4525	{
4526	/*
4527	* We can clear the inhibit force flag as even if we go back to the recompiler
4528	* without executing guest code in VT-x, the flag's condition to be cleared is
4529	* met and thus the cleared state is correct.
4530	*/
4531	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4532	}
4533	}
4534
4535	/*
4536	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4537	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4538	* setting this would block host-NMIs and IRET will not clear the blocking.
4539	*
4540	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4541	*
4542	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4543	*/
4544	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4545	&& CPUMIsGuestNmiBlocking(pVCpu))
4546	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4547
4548	return fIntrState;
4549	}
4550
4551
4552	/**
4553	* Exports the exception intercepts required for guest execution in the VMCS.
4554	*
4555	* @returns VBox status code.
4556	* @param pVCpu The cross context virtual CPU structure.
4557	* @param pVmxTransient The VMX-transient structure.
4558	*
4559	* @remarks No-long-jump zone!!!
4560	*/
4561	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4562	{
4563	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4564	{
4565	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4566	if ( !pVmxTransient->fIsNestedGuest
4567	&& pVCpu->hm.s.fGIMTrapXcptUD)
4568	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4569	else
4570	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4571
4572	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4573	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4574	}
4575	return VINF_SUCCESS;
4576	}
4577
4578
4579	/**
4580	* Exports the guest's RIP into the guest-state area in the VMCS.
4581	*
4582	* @returns VBox status code.
4583	* @param pVCpu The cross context virtual CPU structure.
4584	*
4585	* @remarks No-long-jump zone!!!
4586	*/
4587	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4588	{
4589	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4590	{
4591	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4592
4593	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4594	AssertRCReturn(rc, rc);
4595
4596	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4597	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4598	}
4599	return VINF_SUCCESS;
4600	}
4601
4602
4603	/**
4604	* Exports the guest's RSP into the guest-state area in the VMCS.
4605	*
4606	* @returns VBox status code.
4607	* @param pVCpu The cross context virtual CPU structure.
4608	*
4609	* @remarks No-long-jump zone!!!
4610	*/
4611	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4612	{
4613	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4614	{
4615	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4616
4617	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4618	AssertRCReturn(rc, rc);
4619
4620	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4621	}
4622	return VINF_SUCCESS;
4623	}
4624
4625
4626	/**
4627	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4628	*
4629	* @returns VBox status code.
4630	* @param pVCpu The cross context virtual CPU structure.
4631	* @param pVmxTransient The VMX-transient structure.
4632	*
4633	* @remarks No-long-jump zone!!!
4634	*/
4635	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4636	{
4637	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4638	{
4639	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4640
4641	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4642	Let us assert it as such and use 32-bit VMWRITE. */
4643	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4644	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4645	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4646	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4647
4648	/*
4649	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4650	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4651	* can run the real-mode guest code under Virtual 8086 mode.
4652	*/
4653	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4654	if (pVmcsInfo->RealMode.fRealOnV86Active)
4655	{
4656	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4657	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4658	Assert(!pVmxTransient->fIsNestedGuest);
4659	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4660	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4661	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4662	}
4663
4664	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4665	AssertRCReturn(rc, rc);
4666
4667	/*
4668	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4669	*
4670	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4671	* through the hypervisor debugger using EFLAGS.TF.
4672	*/
4673	if ( !pVmxTransient->fIsNestedGuest
4674	&& !pVCpu->hm.s.fSingleInstruction
4675	&& fEFlags.Bits.u1TF)
4676	{
4677	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4678	* premature trips to ring-3 esp since IEM does not yet handle it. */
4679	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4680	AssertRCReturn(rc, rc);
4681	}
4682	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4683	* nested-guest VMCS. */
4684
4685	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4686	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4687	}
4688	return VINF_SUCCESS;
4689	}
4690
4691
4692	/**
4693	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4694	*
4695	* The guest FPU state is always pre-loaded hence we don't need to bother about
4696	* sharing FPU related CR0 bits between the guest and host.
4697	*
4698	* @returns VBox status code.
4699	* @param pVCpu The cross context virtual CPU structure.
4700	* @param pVmxTransient The VMX-transient structure.
4701	*
4702	* @remarks No-long-jump zone!!!
4703	*/
4704	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4705	{
4706	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4707	{
4708	PVM pVM = pVCpu->CTX_SUFF(pVM);
4709	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4710
4711	/*
4712	* Figure out fixed CR0 bits in VMX operation.
4713	*/
4714	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4715	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4716	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4717	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4718	else
4719	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4720
4721	if (!pVmxTransient->fIsNestedGuest)
4722	{
4723	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4724	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4725	uint64_t const u64ShadowCr0 = u64GuestCr0;
4726	Assert(!RT_HI_U32(u64GuestCr0));
4727
4728	/*
4729	* Setup VT-x's view of the guest CR0.
4730	*/
4731	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4732	if (pVM->hm.s.fNestedPaging)
4733	{
4734	if (CPUMIsGuestPagingEnabled(pVCpu))
4735	{
4736	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4737	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4738	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4739	}
4740	else
4741	{
4742	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4743	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4744	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4745	}
4746
4747	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4748	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4749	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4750	}
4751	else
4752	{
4753	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4754	u64GuestCr0 \|= X86_CR0_WP;
4755	}
4756
4757	/*
4758	* Guest FPU bits.
4759	*
4760	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4761	* using CR0.TS.
4762	*
4763	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4764	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4765	*/
4766	u64GuestCr0 \|= X86_CR0_NE;
4767
4768	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4769	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4770
4771	/*
4772	* Update exception intercepts.
4773	*/
4774	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4775	if (pVmcsInfo->RealMode.fRealOnV86Active)
4776	{
4777	Assert(PDMVmmDevHeapIsEnabled(pVM));
4778	Assert(pVM->hm.s.vmx.pRealModeTSS);
4779	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4780	}
4781	else
4782	{
4783	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4784	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4785	if (fInterceptMF)
4786	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4787	}
4788
4789	/* Additional intercepts for debugging, define these yourself explicitly. */
4790	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4791	uXcptBitmap \|= 0
4792	\| RT_BIT(X86_XCPT_BP)
4793	\| RT_BIT(X86_XCPT_DE)
4794	\| RT_BIT(X86_XCPT_NM)
4795	\| RT_BIT(X86_XCPT_TS)
4796	\| RT_BIT(X86_XCPT_UD)
4797	\| RT_BIT(X86_XCPT_NP)
4798	\| RT_BIT(X86_XCPT_SS)
4799	\| RT_BIT(X86_XCPT_GP)
4800	\| RT_BIT(X86_XCPT_PF)
4801	\| RT_BIT(X86_XCPT_MF)
4802	;
4803	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4804	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4805	#endif
4806	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4807	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4808	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4809
4810	/* Apply the fixed CR0 bits and enable caching. */
4811	u64GuestCr0 \|= fSetCr0;
4812	u64GuestCr0 &= fZapCr0;
4813	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4814
4815	/* Commit the CR0 and related fields to the guest VMCS. */
4816	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4817	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4818	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4819	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4820	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4821	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4822	AssertRCReturn(rc, rc);
4823
4824	/* Update our caches. */
4825	pVmcsInfo->u32ProcCtls = uProcCtls;
4826	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4827
4828	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4829	}
4830	else
4831	{
4832	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4833	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4834	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4835	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4836	Assert(!RT_HI_U32(u64GuestCr0));
4837	Assert(u64GuestCr0 & X86_CR0_NE);
4838
4839	/* Apply the fixed CR0 bits and enable caching. */
4840	u64GuestCr0 \|= fSetCr0;
4841	u64GuestCr0 &= fZapCr0;
4842	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4843
4844	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4845	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4846	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4847	AssertRCReturn(rc, rc);
4848
4849	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4850	}
4851
4852	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4853	}
4854
4855	return VINF_SUCCESS;
4856	}
4857
4858
4859	/**
4860	* Exports the guest control registers (CR3, CR4) into the guest-state area
4861	* in the VMCS.
4862	*
4863	* @returns VBox strict status code.
4864	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4865	* without unrestricted guest access and the VMMDev is not presently
4866	* mapped (e.g. EFI32).
4867	*
4868	* @param pVCpu The cross context virtual CPU structure.
4869	* @param pVmxTransient The VMX-transient structure.
4870	*
4871	* @remarks No-long-jump zone!!!
4872	*/
4873	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4874	{
4875	int rc = VINF_SUCCESS;
4876	PVM pVM = pVCpu->CTX_SUFF(pVM);
4877
4878	/*
4879	* Guest CR2.
4880	* It's always loaded in the assembler code. Nothing to do here.
4881	*/
4882
4883	/*
4884	* Guest CR3.
4885	*/
4886	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4887	{
4888	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4889
4890	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4891	if (pVM->hm.s.fNestedPaging)
4892	{
4893	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4894	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4895
4896	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4897	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4898	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4899	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4900
4901	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4902	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4903	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4904
4905	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4906	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4907	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4908	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4909	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4910	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4911	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4912
4913	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4914	AssertRCReturn(rc, rc);
4915
4916	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4917	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4918	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4919	{
4920	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4921	if (CPUMIsGuestInPAEModeEx(pCtx))
4922	{
4923	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4924	AssertRCReturn(rc, rc);
4925	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4926	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4927	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4928	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4929	AssertRCReturn(rc, rc);
4930	}
4931
4932	/*
4933	* The guest's view of its CR3 is unblemished with nested paging when the
4934	* guest is using paging or we have unrestricted guest execution to handle
4935	* the guest when it's not using paging.
4936	*/
4937	GCPhysGuestCR3 = pCtx->cr3;
4938	}
4939	else
4940	{
4941	/*
4942	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4943	* thinks it accesses physical memory directly, we use our identity-mapped
4944	* page table to map guest-linear to guest-physical addresses. EPT takes care
4945	* of translating it to host-physical addresses.
4946	*/
4947	RTGCPHYS GCPhys;
4948	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4949
4950	/* We obtain it here every time as the guest could have relocated this PCI region. */
4951	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4952	if (RT_SUCCESS(rc))
4953	{ /* likely */ }
4954	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4955	{
4956	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4957	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4958	}
4959	else
4960	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4961
4962	GCPhysGuestCR3 = GCPhys;
4963	}
4964
4965	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4966	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4967	AssertRCReturn(rc, rc);
4968	}
4969	else
4970	{
4971	/* Non-nested paging case, just use the hypervisor's CR3. */
4972	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4973
4974	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4975	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4976	AssertRCReturn(rc, rc);
4977	}
4978
4979	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4980	}
4981
4982	/*
4983	* Guest CR4.
4984	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4985	*/
4986	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4987	{
4988	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4989	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4990	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4991
4992	/*
4993	* Figure out fixed CR4 bits in VMX operation.
4994	*/
4995	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4996	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4997
4998	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4999	uint64_t u64GuestCr4 = pCtx->cr4;
5000	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
5001	Assert(!RT_HI_U32(u64GuestCr4));
5002
5003	/*
5004	* Setup VT-x's view of the guest CR4.
5005	*
5006	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5007	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5008	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5009	*
5010	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5011	*/
5012	if (pVmcsInfo->RealMode.fRealOnV86Active)
5013	{
5014	Assert(pVM->hm.s.vmx.pRealModeTSS);
5015	Assert(PDMVmmDevHeapIsEnabled(pVM));
5016	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5017	}
5018
5019	if (pVM->hm.s.fNestedPaging)
5020	{
5021	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5022	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5023	{
5024	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5025	u64GuestCr4 \|= X86_CR4_PSE;
5026	/* Our identity mapping is a 32-bit page directory. */
5027	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5028	}
5029	/* else use guest CR4.*/
5030	}
5031	else
5032	{
5033	Assert(!pVmxTransient->fIsNestedGuest);
5034
5035	/*
5036	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5037	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5038	*/
5039	switch (pVCpu->hm.s.enmShadowMode)
5040	{
5041	case PGMMODE_REAL: /* Real-mode. */
5042	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5043	case PGMMODE_32_BIT: /* 32-bit paging. */
5044	{
5045	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5046	break;
5047	}
5048
5049	case PGMMODE_PAE: /* PAE paging. */
5050	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5051	{
5052	u64GuestCr4 \|= X86_CR4_PAE;
5053	break;
5054	}
5055
5056	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5057	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5058	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5059	break;
5060	#endif
5061	default:
5062	AssertFailed();
5063	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5064	}
5065	}
5066
5067	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5068	u64GuestCr4 \|= fSetCr4;
5069	u64GuestCr4 &= fZapCr4;
5070
5071	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5072	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4); /** @todo Fix to 64-bit when we drop 32-bit. */
5073	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5074	AssertRCReturn(rc, rc);
5075
5076	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5077	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5078
5079	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5080
5081	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5082	}
5083	return rc;
5084	}
5085
5086
5087	/**
5088	* Exports the guest debug registers into the guest-state area in the VMCS.
5089	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5090	*
5091	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5092	*
5093	* @returns VBox status code.
5094	* @param pVCpu The cross context virtual CPU structure.
5095	* @param pVmxTransient The VMX-transient structure.
5096	*
5097	* @remarks No-long-jump zone!!!
5098	*/
5099	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5100	{
5101	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5102
5103	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5104	* stepping. */
5105	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5106	if (pVmxTransient->fIsNestedGuest)
5107	{
5108	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5109	AssertRCReturn(rc, rc);
5110	return VINF_SUCCESS;
5111	}
5112
5113	#ifdef VBOX_STRICT
5114	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5115	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5116	{
5117	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5118	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5119	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5120	}
5121	#endif
5122
5123	bool fSteppingDB = false;
5124	bool fInterceptMovDRx = false;
5125	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5126	if (pVCpu->hm.s.fSingleInstruction)
5127	{
5128	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5129	PVM pVM = pVCpu->CTX_SUFF(pVM);
5130	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5131	{
5132	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5133	Assert(fSteppingDB == false);
5134	}
5135	else
5136	{
5137	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5138	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5139	pVCpu->hm.s.fClearTrapFlag = true;
5140	fSteppingDB = true;
5141	}
5142	}
5143
5144	uint32_t u32GuestDr7;
5145	if ( fSteppingDB
5146	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5147	{
5148	/*
5149	* Use the combined guest and host DRx values found in the hypervisor register set
5150	* because the hypervisor debugger has breakpoints active or someone is single stepping
5151	* on the host side without a monitor trap flag.
5152	*
5153	* Note! DBGF expects a clean DR6 state before executing guest code.
5154	*/
5155	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5156	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5157	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5158	{
5159	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5160	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5161	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5162	}
5163	else
5164	#endif
5165	if (!CPUMIsHyperDebugStateActive(pVCpu))
5166	{
5167	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5168	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5169	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5170	}
5171
5172	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5173	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5174	pVCpu->hm.s.fUsingHyperDR7 = true;
5175	fInterceptMovDRx = true;
5176	}
5177	else
5178	{
5179	/*
5180	* If the guest has enabled debug registers, we need to load them prior to
5181	* executing guest code so they'll trigger at the right time.
5182	*/
5183	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5184	{
5185	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5186	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5187	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5188	{
5189	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5190	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5191	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5192	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5193	}
5194	else
5195	#endif
5196	if (!CPUMIsGuestDebugStateActive(pVCpu))
5197	{
5198	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5199	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5200	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5201	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5202	}
5203	Assert(!fInterceptMovDRx);
5204	}
5205	/*
5206	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5207	* must intercept #DB in order to maintain a correct DR6 guest value, and
5208	* because we need to intercept it to prevent nested #DBs from hanging the
5209	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5210	*/
5211	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5212	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5213	&& !CPUMIsGuestDebugStateActive(pVCpu))
5214	#else
5215	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5216	#endif
5217	{
5218	fInterceptMovDRx = true;
5219	}
5220
5221	/* Update DR7 with the actual guest value. */
5222	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5223	pVCpu->hm.s.fUsingHyperDR7 = false;
5224	}
5225
5226	if (fInterceptMovDRx)
5227	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5228	else
5229	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5230
5231	/*
5232	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5233	* monitor-trap flag and update our cache.
5234	*/
5235	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5236	{
5237	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5238	AssertRCReturn(rc2, rc2);
5239	pVmcsInfo->u32ProcCtls = uProcCtls;
5240	}
5241
5242	/*
5243	* Update guest DR7.
5244	*/
5245	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5246	AssertRCReturn(rc, rc);
5247
5248	/*
5249	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5250	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5251	*
5252	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5253	*/
5254	if (fSteppingDB)
5255	{
5256	Assert(pVCpu->hm.s.fSingleInstruction);
5257	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5258
5259	uint32_t fIntrState = 0;
5260	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5261	AssertRCReturn(rc, rc);
5262
5263	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5264	{
5265	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5266	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5267	AssertRCReturn(rc, rc);
5268	}
5269	}
5270
5271	return VINF_SUCCESS;
5272	}
5273
5274
5275	#ifdef VBOX_STRICT
5276	/**
5277	* Strict function to validate segment registers.
5278	*
5279	* @param pVCpu The cross context virtual CPU structure.
5280	* @param pVmcsInfo The VMCS info. object.
5281	*
5282	* @remarks Will import guest CR0 on strict builds during validation of
5283	* segments.
5284	*/
5285	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5286	{
5287	/*
5288	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5289	*
5290	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5291	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5292	* unusable bit and doesn't change the guest-context value.
5293	*/
5294	PVM pVM = pVCpu->CTX_SUFF(pVM);
5295	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5296	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5297	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5298	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5299	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5300	{
5301	/* Protected mode checks */
5302	/* CS */
5303	Assert(pCtx->cs.Attr.n.u1Present);
5304	Assert(!(pCtx->cs.Attr.u & 0xf00));
5305	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5306	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5307	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5308	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5309	\|\| (pCtx->cs.Attr.n.u1Granularity));
5310	/* CS cannot be loaded with NULL in protected mode. */
5311	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5312	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5313	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5314	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5315	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5316	else
5317	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5318	/* SS */
5319	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5320	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5321	if ( !(pCtx->cr0 & X86_CR0_PE)
5322	\|\| pCtx->cs.Attr.n.u4Type == 3)
5323	{
5324	Assert(!pCtx->ss.Attr.n.u2Dpl);
5325	}
5326	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5327	{
5328	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5329	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5330	Assert(pCtx->ss.Attr.n.u1Present);
5331	Assert(!(pCtx->ss.Attr.u & 0xf00));
5332	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5333	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5334	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5335	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5336	\|\| (pCtx->ss.Attr.n.u1Granularity));
5337	}
5338	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5339	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5340	{
5341	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5342	Assert(pCtx->ds.Attr.n.u1Present);
5343	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5344	Assert(!(pCtx->ds.Attr.u & 0xf00));
5345	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5346	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5347	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5348	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5349	\|\| (pCtx->ds.Attr.n.u1Granularity));
5350	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5351	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5352	}
5353	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5354	{
5355	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5356	Assert(pCtx->es.Attr.n.u1Present);
5357	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5358	Assert(!(pCtx->es.Attr.u & 0xf00));
5359	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5360	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5361	\|\| !(pCtx->es.Attr.n.u1Granularity));
5362	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5363	\|\| (pCtx->es.Attr.n.u1Granularity));
5364	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5365	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5366	}
5367	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5368	{
5369	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5370	Assert(pCtx->fs.Attr.n.u1Present);
5371	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5372	Assert(!(pCtx->fs.Attr.u & 0xf00));
5373	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5374	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5375	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5376	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5377	\|\| (pCtx->fs.Attr.n.u1Granularity));
5378	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5379	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5380	}
5381	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5382	{
5383	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5384	Assert(pCtx->gs.Attr.n.u1Present);
5385	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5386	Assert(!(pCtx->gs.Attr.u & 0xf00));
5387	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5388	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5389	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5390	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5391	\|\| (pCtx->gs.Attr.n.u1Granularity));
5392	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5393	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5394	}
5395	/* 64-bit capable CPUs. */
5396	# if HC_ARCH_BITS == 64
5397	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5398	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5399	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5400	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5401	# endif
5402	}
5403	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5404	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5405	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5406	{
5407	/* Real and v86 mode checks. */
5408	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5409	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5410	if (pVmcsInfo->RealMode.fRealOnV86Active)
5411	{
5412	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5413	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5414	}
5415	else
5416	{
5417	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5418	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5419	}
5420
5421	/* CS */
5422	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5423	Assert(pCtx->cs.u32Limit == 0xffff);
5424	Assert(u32CSAttr == 0xf3);
5425	/* SS */
5426	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5427	Assert(pCtx->ss.u32Limit == 0xffff);
5428	Assert(u32SSAttr == 0xf3);
5429	/* DS */
5430	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5431	Assert(pCtx->ds.u32Limit == 0xffff);
5432	Assert(u32DSAttr == 0xf3);
5433	/* ES */
5434	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5435	Assert(pCtx->es.u32Limit == 0xffff);
5436	Assert(u32ESAttr == 0xf3);
5437	/* FS */
5438	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5439	Assert(pCtx->fs.u32Limit == 0xffff);
5440	Assert(u32FSAttr == 0xf3);
5441	/* GS */
5442	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5443	Assert(pCtx->gs.u32Limit == 0xffff);
5444	Assert(u32GSAttr == 0xf3);
5445	/* 64-bit capable CPUs. */
5446	# if HC_ARCH_BITS == 64
5447	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5448	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5449	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5450	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5451	# endif
5452	}
5453	}
5454	#endif /* VBOX_STRICT */
5455
5456
5457	/**
5458	* Exports a guest segment register into the guest-state area in the VMCS.
5459	*
5460	* @returns VBox status code.
5461	* @param pVCpu The cross context virtual CPU structure.
5462	* @param pVmcsInfo The VMCS info. object.
5463	* @param iSegReg The segment register number (X86_SREG_XXX).
5464	* @param pSelReg Pointer to the segment selector.
5465	*
5466	* @remarks No-long-jump zone!!!
5467	*/
5468	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5469	{
5470	Assert(iSegReg < X86_SREG_COUNT);
5471	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5472	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5473	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5474	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5475
5476	uint32_t u32Access = pSelReg->Attr.u;
5477	if (pVmcsInfo->RealMode.fRealOnV86Active)
5478	{
5479	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5480	u32Access = 0xf3;
5481	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5482	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5483	RT_NOREF_PV(pVCpu);
5484	}
5485	else
5486	{
5487	/*
5488	* The way to differentiate between whether this is really a null selector or was just
5489	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5490	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5491	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5492	* NULL selectors loaded in protected-mode have their attribute as 0.
5493	*/
5494	if (!u32Access)
5495	u32Access = X86DESCATTR_UNUSABLE;
5496	}
5497
5498	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5499	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5500	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5501
5502	/*
5503	* Commit it to the VMCS.
5504	*/
5505	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5506	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5507	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5508	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5509	AssertRCReturn(rc, rc);
5510	return rc;
5511	}
5512
5513
5514	/**
5515	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5516	* area in the VMCS.
5517	*
5518	* @returns VBox status code.
5519	* @param pVCpu The cross context virtual CPU structure.
5520	* @param pVmxTransient The VMX-transient structure.
5521	*
5522	* @remarks Will import guest CR0 on strict builds during validation of
5523	* segments.
5524	* @remarks No-long-jump zone!!!
5525	*/
5526	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5527	{
5528	int rc = VERR_INTERNAL_ERROR_5;
5529	PVM pVM = pVCpu->CTX_SUFF(pVM);
5530	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5531	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5532
5533	/*
5534	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5535	*/
5536	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5537	{
5538	#ifdef VBOX_WITH_REM
5539	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5540	{
5541	Assert(!pVmxTransient->fIsNestedGuest);
5542	Assert(pVM->hm.s.vmx.pRealModeTSS);
5543	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5544	if ( pVmcsInfo->fWasInRealMode
5545	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5546	{
5547	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5548	in real-mode (e.g. OpenBSD 4.0) */
5549	REMFlushTBs(pVM);
5550	Log4Func(("Switch to protected mode detected!\n"));
5551	pVmcsInfo->fWasInRealMode = false;
5552	}
5553	}
5554	#endif
5555	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5556	{
5557	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5558	if (pVmcsInfo->RealMode.fRealOnV86Active)
5559	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5560	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5561	AssertRCReturn(rc, rc);
5562	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5563	}
5564
5565	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5566	{
5567	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5568	if (pVmcsInfo->RealMode.fRealOnV86Active)
5569	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5570	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5571	AssertRCReturn(rc, rc);
5572	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5573	}
5574
5575	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5576	{
5577	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5578	if (pVmcsInfo->RealMode.fRealOnV86Active)
5579	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5580	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5581	AssertRCReturn(rc, rc);
5582	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5583	}
5584
5585	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5586	{
5587	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5588	if (pVmcsInfo->RealMode.fRealOnV86Active)
5589	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5590	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5591	AssertRCReturn(rc, rc);
5592	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5593	}
5594
5595	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5596	{
5597	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5598	if (pVmcsInfo->RealMode.fRealOnV86Active)
5599	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5600	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5601	AssertRCReturn(rc, rc);
5602	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5603	}
5604
5605	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5606	{
5607	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5608	if (pVmcsInfo->RealMode.fRealOnV86Active)
5609	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5610	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5611	AssertRCReturn(rc, rc);
5612	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5613	}
5614
5615	#ifdef VBOX_STRICT
5616	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5617	#endif
5618	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5619	pCtx->cs.Attr.u));
5620	}
5621
5622	/*
5623	* Guest TR.
5624	*/
5625	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5626	{
5627	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5628
5629	/*
5630	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5631	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5632	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5633	*/
5634	uint16_t u16Sel;
5635	uint32_t u32Limit;
5636	uint64_t u64Base;
5637	uint32_t u32AccessRights;
5638	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5639	{
5640	u16Sel = pCtx->tr.Sel;
5641	u32Limit = pCtx->tr.u32Limit;
5642	u64Base = pCtx->tr.u64Base;
5643	u32AccessRights = pCtx->tr.Attr.u;
5644	}
5645	else
5646	{
5647	Assert(!pVmxTransient->fIsNestedGuest);
5648	Assert(pVM->hm.s.vmx.pRealModeTSS);
5649	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5650
5651	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5652	RTGCPHYS GCPhys;
5653	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5654	AssertRCReturn(rc, rc);
5655
5656	X86DESCATTR DescAttr;
5657	DescAttr.u = 0;
5658	DescAttr.n.u1Present = 1;
5659	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5660
5661	u16Sel = 0;
5662	u32Limit = HM_VTX_TSS_SIZE;
5663	u64Base = GCPhys;
5664	u32AccessRights = DescAttr.u;
5665	}
5666
5667	/* Validate. */
5668	Assert(!(u16Sel & RT_BIT(2)));
5669	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5670	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5671	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5672	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5673	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5674	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5675	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5676	Assert( (u32Limit & 0xfff) == 0xfff
5677	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5678	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5679	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5680
5681	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5682	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5683	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5684	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5685	AssertRCReturn(rc, rc);
5686
5687	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5688	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5689	}
5690
5691	/*
5692	* Guest GDTR.
5693	*/
5694	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5695	{
5696	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5697
5698	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5699	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5700	AssertRCReturn(rc, rc);
5701
5702	/* Validate. */
5703	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5704
5705	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5706	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5707	}
5708
5709	/*
5710	* Guest LDTR.
5711	*/
5712	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5713	{
5714	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5715
5716	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5717	uint32_t u32Access;
5718	if ( !pVmxTransient->fIsNestedGuest
5719	&& !pCtx->ldtr.Attr.u)
5720	u32Access = X86DESCATTR_UNUSABLE;
5721	else
5722	u32Access = pCtx->ldtr.Attr.u;
5723
5724	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5725	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5726	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5727	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5728	AssertRCReturn(rc, rc);
5729
5730	/* Validate. */
5731	if (!(u32Access & X86DESCATTR_UNUSABLE))
5732	{
5733	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5734	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5735	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5736	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5737	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5738	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5739	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5740	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5741	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5742	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5743	}
5744
5745	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5746	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5747	}
5748
5749	/*
5750	* Guest IDTR.
5751	*/
5752	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5753	{
5754	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5755
5756	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5757	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5758	AssertRCReturn(rc, rc);
5759
5760	/* Validate. */
5761	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5762
5763	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5764	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5765	}
5766
5767	return VINF_SUCCESS;
5768	}
5769
5770
5771	/**
5772	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5773	* areas.
5774	*
5775	* These MSRs will automatically be loaded to the host CPU on every successful
5776	* VM-entry and stored from the host CPU on every successful VM-exit.
5777	*
5778	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5779	* actual host MSR values are not- updated here for performance reasons. See
5780	* hmR0VmxExportHostMsrs().
5781	*
5782	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5783	*
5784	* @returns VBox status code.
5785	* @param pVCpu The cross context virtual CPU structure.
5786	* @param pVmxTransient The VMX-transient structure.
5787	*
5788	* @remarks No-long-jump zone!!!
5789	*/
5790	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5791	{
5792	AssertPtr(pVCpu);
5793	AssertPtr(pVmxTransient);
5794
5795	PVM pVM = pVCpu->CTX_SUFF(pVM);
5796	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5797
5798	/*
5799	* MSRs that we use the auto-load/store MSR area in the VMCS.
5800	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5801	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5802	*
5803	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5804	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5805	* emulation, nothing to do here.
5806	*/
5807	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5808	{
5809	if ( !pVmxTransient->fIsNestedGuest
5810	&& pVM->hm.s.fAllow64BitGuests)
5811	{
5812	#if HC_ARCH_BITS == 32
5813	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5814	Assert(!pVmxTransient->fIsNestedGuest);
5815
5816	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5817	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5818	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5819	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5820	AssertRCReturn(rc, rc);
5821	#endif
5822	}
5823	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5824	}
5825
5826	/*
5827	* Guest Sysenter MSRs.
5828	*/
5829	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5830	{
5831	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5832
5833	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5834	{
5835	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5836	AssertRCReturn(rc, rc);
5837	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5838	}
5839
5840	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5841	{
5842	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5843	AssertRCReturn(rc, rc);
5844	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5845	}
5846
5847	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5848	{
5849	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5850	AssertRCReturn(rc, rc);
5851	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5852	}
5853	}
5854
5855	/*
5856	* Guest/host EFER MSR.
5857	*/
5858	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5859	{
5860	/* Whether we are using the VMCS to swap the EFER MSR must have been
5861	determined earlier while exporting VM-entry/VM-exit controls. */
5862	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5863	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5864
5865	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5866	{
5867	/*
5868	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5869	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5870	*/
5871	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5872	{
5873	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5874	AssertRCReturn(rc, rc);
5875	}
5876	else
5877	{
5878	/*
5879	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5880	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5881	*/
5882	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5883	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5884	AssertRCReturn(rc, rc);
5885	}
5886	}
5887	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5888	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5889
5890	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5891	}
5892
5893	/*
5894	* Other MSRs.
5895	* Speculation Control (R/W).
5896	*/
5897	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5898	{
5899	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5900	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5901	{
5902	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5903	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5904	AssertRCReturn(rc, rc);
5905	}
5906	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5907	}
5908
5909	return VINF_SUCCESS;
5910	}
5911
5912
5913	/**
5914	* Selects up the appropriate function to run guest code.
5915	*
5916	* @returns VBox status code.
5917	* @param pVCpu The cross context virtual CPU structure.
5918	* @param pVmxTransient The VMX-transient structure.
5919	*
5920	* @remarks No-long-jump zone!!!
5921	*/
5922	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5923	{
5924	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5925	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5926
5927	if (CPUMIsGuestInLongModeEx(pCtx))
5928	{
5929	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5930	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5931	#endif
5932	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5933	#if HC_ARCH_BITS == 32
5934	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5935	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5936	{
5937	#ifdef VBOX_STRICT
5938	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5939	{
5940	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5941	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5942	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5943	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5944	("fCtxChanged=%#RX64\n", fCtxChanged));
5945	}
5946	#endif
5947	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5948
5949	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5950	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5951	pVmcsInfo->fSwitchedTo64on32 = true;
5952	Log4Func(("Selected 64-bit switcher\n"));
5953	}
5954	#else
5955	/* 64-bit host. */
5956	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5957	#endif
5958	}
5959	else
5960	{
5961	/* Guest is not in long mode, use the 32-bit handler. */
5962	#if HC_ARCH_BITS == 32
5963	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5964	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5965	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5966	{
5967	# ifdef VBOX_STRICT
5968	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5969	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5970	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5971	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5972	("fCtxChanged=%#RX64\n", fCtxChanged));
5973	# endif
5974	}
5975	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5976	/*
5977	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5978	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5979	* switcher flag now because we know the guest is in a sane state where it's safe
5980	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5981	* the much faster 32-bit switcher again.
5982	*/
5983	if (!pVmcsInfo->fSwitchedTo64on32)
5984	{
5985	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5986	Log4Func(("Selected 32-bit switcher\n"));
5987	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5988	}
5989	else
5990	{
5991	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5992	if ( pVmcsInfo->RealMode.fRealOnV86Active
5993	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5994	{
5995	pVmcsInfo->fSwitchedTo64on32 = false;
5996	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5997	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5998	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5999	\| HM_CHANGED_HOST_CONTEXT);
6000	Log4Func(("Selected 32-bit switcher (safe)\n"));
6001	}
6002	}
6003	# else
6004	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6005	# endif
6006	#else
6007	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6008	#endif
6009	}
6010	Assert(pVmcsInfo->pfnStartVM);
6011	return VINF_SUCCESS;
6012	}
6013
6014
6015	/**
6016	* Wrapper for running the guest code in VT-x.
6017	*
6018	* @returns VBox status code, no informational status codes.
6019	* @param pVCpu The cross context virtual CPU structure.
6020	* @param pVmxTransient The VMX-transient structure.
6021	*
6022	* @remarks No-long-jump zone!!!
6023	*/
6024	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6025	{
6026	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6027	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6028	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6029
6030	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6031
6032	/*
6033	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6034	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6035	* callee-saved and thus the need for this XMM wrapper.
6036	*
6037	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6038	*/
6039	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6040	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6041	PVM pVM = pVCpu->CTX_SUFF(pVM);
6042	#ifdef VBOX_WITH_KERNEL_USING_XMM
6043	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6044	#else
6045	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6046	#endif
6047	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6048	return rc;
6049	}
6050
6051
6052	/**
6053	* Reports world-switch error and dumps some useful debug info.
6054	*
6055	* @param pVCpu The cross context virtual CPU structure.
6056	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6057	* @param pVmxTransient The VMX-transient structure (only
6058	* exitReason updated).
6059	*/
6060	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6061	{
6062	Assert(pVCpu);
6063	Assert(pVmxTransient);
6064	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6065
6066	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6067	switch (rcVMRun)
6068	{
6069	case VERR_VMX_INVALID_VMXON_PTR:
6070	AssertFailed();
6071	break;
6072	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6073	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6074	{
6075	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6076	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6077	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6078	AssertRC(rc);
6079
6080	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6081	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6082	Cannot do it here as we may have been long preempted. */
6083
6084	#ifdef VBOX_STRICT
6085	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6086	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6087	pVmxTransient->uExitReason));
6088	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6089	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6090	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6091	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6092	else
6093	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6094	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6095	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6096
6097	/* VMX control bits. */
6098	uint32_t u32Val;
6099	uint64_t u64Val;
6100	RTHCUINTREG uHCReg;
6101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6102	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6104	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6105	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6106	{
6107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6108	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6109	}
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6113	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6115	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6117	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6119	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6121	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6123	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6125	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6127	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6129	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6131	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6132	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6133	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6135	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6136	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6137	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6138	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6139	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6140	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6141	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6142	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6143	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6144	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6145	{
6146	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6147	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6148	}
6149
6150	/* Guest bits. */
6151	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6152	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6153	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6154	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6155	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6156	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6157	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6158	{
6159	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6160	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6161	}
6162
6163	/* Host bits. */
6164	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6165	Log4(("Host CR0 %#RHr\n", uHCReg));
6166	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6167	Log4(("Host CR3 %#RHr\n", uHCReg));
6168	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6169	Log4(("Host CR4 %#RHr\n", uHCReg));
6170
6171	RTGDTR HostGdtr;
6172	PCX86DESCHC pDesc;
6173	ASMGetGDTR(&HostGdtr);
6174	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6175	Log4(("Host CS %#08x\n", u32Val));
6176	if (u32Val < HostGdtr.cbGdt)
6177	{
6178	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6179	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6180	}
6181
6182	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6183	Log4(("Host DS %#08x\n", u32Val));
6184	if (u32Val < HostGdtr.cbGdt)
6185	{
6186	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6187	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6188	}
6189
6190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6191	Log4(("Host ES %#08x\n", u32Val));
6192	if (u32Val < HostGdtr.cbGdt)
6193	{
6194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6195	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6196	}
6197
6198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6199	Log4(("Host FS %#08x\n", u32Val));
6200	if (u32Val < HostGdtr.cbGdt)
6201	{
6202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6203	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6204	}
6205
6206	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6207	Log4(("Host GS %#08x\n", u32Val));
6208	if (u32Val < HostGdtr.cbGdt)
6209	{
6210	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6211	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6212	}
6213
6214	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6215	Log4(("Host SS %#08x\n", u32Val));
6216	if (u32Val < HostGdtr.cbGdt)
6217	{
6218	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6219	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6220	}
6221
6222	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6223	Log4(("Host TR %#08x\n", u32Val));
6224	if (u32Val < HostGdtr.cbGdt)
6225	{
6226	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6227	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6228	}
6229
6230	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6231	Log4(("Host TR Base %#RHv\n", uHCReg));
6232	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6233	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6234	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6235	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6236	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6237	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6238	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6239	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6240	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6241	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6242	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6243	Log4(("Host RSP %#RHv\n", uHCReg));
6244	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6245	Log4(("Host RIP %#RHv\n", uHCReg));
6246	# if HC_ARCH_BITS == 64
6247	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6248	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6249	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6250	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6251	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6252	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6253	# endif
6254	#endif /* VBOX_STRICT */
6255	break;
6256	}
6257
6258	default:
6259	/* Impossible */
6260	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6261	break;
6262	}
6263	}
6264
6265
6266	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6267	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6268	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6269	# endif
6270
6271	/**
6272	* Initialize the VMCS-Read cache.
6273	*
6274	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6275	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6276	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6277	* (those that have a 32-bit FULL & HIGH part).
6278	*
6279	* @param pVCpu The cross context virtual CPU structure.
6280	*/
6281	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6282	{
6283	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6284	do { \
6285	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6286	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6287	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6288	++cReadFields; \
6289	} while (0)
6290
6291	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6292	uint32_t cReadFields = 0;
6293
6294	/*
6295	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6296	* and serve to indicate exceptions to the rules.
6297	*/
6298
6299	/* Guest-natural selector base fields. */
6300	#if 0
6301	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6304	#endif
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6317	#if 0
6318	/* Unused natural width guest-state fields. */
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6321	#endif
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6324
6325	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6326	these 64-bit fields (using "FULL" and "HIGH" fields). */
6327	#if 0
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6334	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6335	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6336	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6337	#endif
6338
6339	/* Natural width guest-state fields. */
6340	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6341	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6342
6343	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6344	{
6345	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6346	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6347	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6348	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6349	}
6350	else
6351	{
6352	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6353	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6354	}
6355
6356	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6357	}
6358
6359
6360	/**
6361	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6362	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6363	* darwin, running 64-bit guests).
6364	*
6365	* @returns VBox status code.
6366	* @param pVCpu The cross context virtual CPU structure.
6367	* @param idxField The VMCS field encoding.
6368	* @param u64Val 16, 32 or 64-bit value.
6369	*/
6370	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6371	{
6372	int rc;
6373	switch (idxField)
6374	{
6375	/*
6376	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6377	*/
6378	/* 64-bit Control fields. */
6379	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6380	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6381	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6382	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6383	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6384	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6385	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6386	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6387	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6388	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6389	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6390	case VMX_VMCS64_CTRL_EPTP_FULL:
6391	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6392	/* 64-bit Guest-state fields. */
6393	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6394	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6395	case VMX_VMCS64_GUEST_PAT_FULL:
6396	case VMX_VMCS64_GUEST_EFER_FULL:
6397	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6398	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6399	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6400	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6401	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6402	/* 64-bit Host-state fields. */
6403	case VMX_VMCS64_HOST_PAT_FULL:
6404	case VMX_VMCS64_HOST_EFER_FULL:
6405	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6406	{
6407	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6408	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6409	break;
6410	}
6411
6412	/*
6413	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6414	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6415	*/
6416	/* Natural-width Guest-state fields. */
6417	case VMX_VMCS_GUEST_CR3:
6418	case VMX_VMCS_GUEST_ES_BASE:
6419	case VMX_VMCS_GUEST_CS_BASE:
6420	case VMX_VMCS_GUEST_SS_BASE:
6421	case VMX_VMCS_GUEST_DS_BASE:
6422	case VMX_VMCS_GUEST_FS_BASE:
6423	case VMX_VMCS_GUEST_GS_BASE:
6424	case VMX_VMCS_GUEST_LDTR_BASE:
6425	case VMX_VMCS_GUEST_TR_BASE:
6426	case VMX_VMCS_GUEST_GDTR_BASE:
6427	case VMX_VMCS_GUEST_IDTR_BASE:
6428	case VMX_VMCS_GUEST_RSP:
6429	case VMX_VMCS_GUEST_RIP:
6430	case VMX_VMCS_GUEST_SYSENTER_ESP:
6431	case VMX_VMCS_GUEST_SYSENTER_EIP:
6432	{
6433	if (!(RT_HI_U32(u64Val)))
6434	{
6435	/* If this field is 64-bit, VT-x will zero out the top bits. */
6436	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6437	}
6438	else
6439	{
6440	/* Assert that only the 32->64 switcher case should ever come here. */
6441	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6442	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6443	}
6444	break;
6445	}
6446
6447	default:
6448	{
6449	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6450	pVCpu->hm.s.u32HMError = idxField;
6451	rc = VERR_INVALID_PARAMETER;
6452	break;
6453	}
6454	}
6455	AssertRCReturn(rc, rc);
6456	return rc;
6457	}
6458
6459
6460	/**
6461	* Queue up a VMWRITE by using the VMCS write cache.
6462	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6463	*
6464	* @param pVCpu The cross context virtual CPU structure.
6465	* @param idxField The VMCS field encoding.
6466	* @param u64Val 16, 32 or 64-bit value.
6467	*/
6468	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6469	{
6470	AssertPtr(pVCpu);
6471	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6472
6473	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6474	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6475
6476	/* Make sure there are no duplicates. */
6477	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6478	{
6479	if (pCache->Write.aField[i] == idxField)
6480	{
6481	pCache->Write.aFieldVal[i] = u64Val;
6482	return VINF_SUCCESS;
6483	}
6484	}
6485
6486	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6487	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6488	pCache->Write.cValidEntries++;
6489	return VINF_SUCCESS;
6490	}
6491	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6492
6493
6494	/**
6495	* Sets up the usage of TSC-offsetting and updates the VMCS.
6496	*
6497	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6498	* VMX-preemption timer.
6499	*
6500	* @returns VBox status code.
6501	* @param pVCpu The cross context virtual CPU structure.
6502	* @param pVmxTransient The VMX-transient structure.
6503	*
6504	* @remarks No-long-jump zone!!!
6505	*/
6506	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6507	{
6508	bool fOffsettedTsc;
6509	bool fParavirtTsc;
6510	uint64_t uTscOffset;
6511	PVM pVM = pVCpu->CTX_SUFF(pVM);
6512	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6513
6514	if (pVM->hm.s.vmx.fUsePreemptTimer)
6515	{
6516	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6517
6518	/* Make sure the returned values have sane upper and lower boundaries. */
6519	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6520	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6521	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6522	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6523
6524	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6525	* preemption timers here. We probably need to clamp the preemption timer,
6526	* after converting the timer value to the host. */
6527	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6528	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6529	AssertRC(rc);
6530	}
6531	else
6532	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6533
6534	if (fParavirtTsc)
6535	{
6536	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6537	information before every VM-entry, hence disable it for performance sake. */
6538	#if 0
6539	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6540	AssertRC(rc);
6541	#endif
6542	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6543	}
6544
6545	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6546	if ( fOffsettedTsc
6547	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6548	{
6549	if (pVmxTransient->fIsNestedGuest)
6550	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6551	if (pVmcsInfo->u64TscOffset != uTscOffset)
6552	{
6553	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6554	AssertRC(rc);
6555	pVmcsInfo->u64TscOffset = uTscOffset;
6556	}
6557
6558	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6559	{
6560	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6561	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6562	AssertRC(rc);
6563	pVmcsInfo->u32ProcCtls = uProcCtls;
6564	}
6565	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6566	}
6567	else
6568	{
6569	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6570	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6571	{
6572	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6573	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6574	AssertRC(rc);
6575	pVmcsInfo->u32ProcCtls = uProcCtls;
6576	}
6577	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6578	}
6579	}
6580
6581
6582	/**
6583	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6584	* VM-exit interruption info type.
6585	*
6586	* @returns The IEM exception flags.
6587	* @param uVector The event vector.
6588	* @param uVmxEventType The VMX event type.
6589	*
6590	* @remarks This function currently only constructs flags required for
6591	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6592	* and CR2 aspects of an exception are not included).
6593	*/
6594	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6595	{
6596	uint32_t fIemXcptFlags;
6597	switch (uVmxEventType)
6598	{
6599	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6600	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6601	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6602	break;
6603
6604	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6605	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6606	break;
6607
6608	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6609	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6610	break;
6611
6612	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6613	{
6614	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6615	if (uVector == X86_XCPT_BP)
6616	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6617	else if (uVector == X86_XCPT_OF)
6618	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6619	else
6620	{
6621	fIemXcptFlags = 0;
6622	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6623	}
6624	break;
6625	}
6626
6627	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6628	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6629	break;
6630
6631	default:
6632	fIemXcptFlags = 0;
6633	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6634	break;
6635	}
6636	return fIemXcptFlags;
6637	}
6638
6639
6640	/**
6641	* Sets an event as a pending event to be injected into the guest.
6642	*
6643	* @param pVCpu The cross context virtual CPU structure.
6644	* @param u32IntInfo The VM-entry interruption-information field.
6645	* @param cbInstr The VM-entry instruction length in bytes (for software
6646	* interrupts, exceptions and privileged software
6647	* exceptions).
6648	* @param u32ErrCode The VM-entry exception error code.
6649	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6650	* page-fault.
6651	*/
6652	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6653	RTGCUINTPTR GCPtrFaultAddress)
6654	{
6655	Assert(!pVCpu->hm.s.Event.fPending);
6656	pVCpu->hm.s.Event.fPending = true;
6657	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6658	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6659	pVCpu->hm.s.Event.cbInstr = cbInstr;
6660	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6661	}
6662
6663
6664	/**
6665	* Sets an external interrupt as pending-for-injection into the VM.
6666	*
6667	* @param pVCpu The cross context virtual CPU structure.
6668	* @param u8Interrupt The external interrupt vector.
6669	*/
6670	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6671	{
6672	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6673	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6674	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6675	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6676	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6677	}
6678
6679
6680	/**
6681	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6682	*
6683	* @param pVCpu The cross context virtual CPU structure.
6684	*/
6685	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6686	{
6687	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6689	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6690	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6691	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6692	}
6693
6694
6695	/**
6696	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6697	*
6698	* @param pVCpu The cross context virtual CPU structure.
6699	*/
6700	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6701	{
6702	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6703	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6704	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6706	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6707	}
6708
6709
6710	/**
6711	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6712	*
6713	* @param pVCpu The cross context virtual CPU structure.
6714	*/
6715	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6716	{
6717	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6718	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6719	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6720	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6721	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6722	}
6723
6724
6725	/**
6726	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6727	*
6728	* @param pVCpu The cross context virtual CPU structure.
6729	*/
6730	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6731	{
6732	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6733	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6734	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6735	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6736	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6737	}
6738
6739
6740	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6741	/**
6742	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6743	*
6744	* @param pVCpu The cross context virtual CPU structure.
6745	* @param u32ErrCode The error code for the general-protection exception.
6746	*/
6747	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6748	{
6749	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6751	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6752	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6753	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6754	}
6755
6756
6757	/**
6758	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6759	*
6760	* @param pVCpu The cross context virtual CPU structure.
6761	* @param u32ErrCode The error code for the stack exception.
6762	*/
6763	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6764	{
6765	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6766	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6767	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6768	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6769	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6770	}
6771
6772
6773	/**
6774	* Decodes the memory operand of an instruction that caused a VM-exit.
6775	*
6776	* The VM-exit qualification field provides the displacement field for memory
6777	* operand instructions, if any.
6778	*
6779	* @returns Strict VBox status code (i.e. informational status codes too).
6780	* @retval VINF_SUCCESS if the operand was successfully decoded.
6781	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6782	* operand.
6783	* @param pVCpu The cross context virtual CPU structure.
6784	* @param uExitInstrInfo The VM-exit instruction information field.
6785	* @param enmMemAccess The memory operand's access type (read or write).
6786	* @param GCPtrDisp The instruction displacement field, if any. For
6787	* RIP-relative addressing pass RIP + displacement here.
6788	* @param pGCPtrMem Where to store the effective destination memory address.
6789	*
6790	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6791	* virtual-8086 mode hence skips those checks while verifying if the
6792	* segment is valid.
6793	*/
6794	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6795	PRTGCPTR pGCPtrMem)
6796	{
6797	Assert(pGCPtrMem);
6798	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6799	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6800	\| CPUMCTX_EXTRN_CR0);
6801
6802	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6803	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6804	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6805
6806	VMXEXITINSTRINFO ExitInstrInfo;
6807	ExitInstrInfo.u = uExitInstrInfo;
6808	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6809	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6810	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6811	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6812	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6813	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6814	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6815	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6816	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6817
6818	/*
6819	* Validate instruction information.
6820	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6821	*/
6822	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6823	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6824	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6825	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6826	AssertLogRelMsgReturn(fIsMemOperand,
6827	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6828
6829	/*
6830	* Compute the complete effective address.
6831	*
6832	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6833	* See AMD spec. 4.5.2 "Segment Registers".
6834	*/
6835	RTGCPTR GCPtrMem = GCPtrDisp;
6836	if (fBaseRegValid)
6837	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6838	if (fIdxRegValid)
6839	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6840
6841	RTGCPTR const GCPtrOff = GCPtrMem;
6842	if ( !fIsLongMode
6843	\|\| iSegReg >= X86_SREG_FS)
6844	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6845	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6846
6847	/*
6848	* Validate effective address.
6849	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6850	*/
6851	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6852	Assert(cbAccess > 0);
6853	if (fIsLongMode)
6854	{
6855	if (X86_IS_CANONICAL(GCPtrMem))
6856	{
6857	*pGCPtrMem = GCPtrMem;
6858	return VINF_SUCCESS;
6859	}
6860
6861	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6862	* "Data Limit Checks in 64-bit Mode". */
6863	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6864	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6865	return VINF_HM_PENDING_XCPT;
6866	}
6867
6868	/*
6869	* This is a watered down version of iemMemApplySegment().
6870	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6871	* and segment CPL/DPL checks are skipped.
6872	*/
6873	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6874	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6875	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6876
6877	/* Check if the segment is present and usable. */
6878	if ( pSel->Attr.n.u1Present
6879	&& !pSel->Attr.n.u1Unusable)
6880	{
6881	Assert(pSel->Attr.n.u1DescType);
6882	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6883	{
6884	/* Check permissions for the data segment. */
6885	if ( enmMemAccess == VMXMEMACCESS_WRITE
6886	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6887	{
6888	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6889	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6890	return VINF_HM_PENDING_XCPT;
6891	}
6892
6893	/* Check limits if it's a normal data segment. */
6894	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6895	{
6896	if ( GCPtrFirst32 > pSel->u32Limit
6897	\|\| GCPtrLast32 > pSel->u32Limit)
6898	{
6899	Log4Func(("Data segment limit exceeded."
6900	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6901	GCPtrLast32, pSel->u32Limit));
6902	if (iSegReg == X86_SREG_SS)
6903	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6904	else
6905	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6906	return VINF_HM_PENDING_XCPT;
6907	}
6908	}
6909	else
6910	{
6911	/* Check limits if it's an expand-down data segment.
6912	Note! The upper boundary is defined by the B bit, not the G bit! */
6913	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6914	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6915	{
6916	Log4Func(("Expand-down data segment limit exceeded."
6917	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6918	GCPtrLast32, pSel->u32Limit));
6919	if (iSegReg == X86_SREG_SS)
6920	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6921	else
6922	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6923	return VINF_HM_PENDING_XCPT;
6924	}
6925	}
6926	}
6927	else
6928	{
6929	/* Check permissions for the code segment. */
6930	if ( enmMemAccess == VMXMEMACCESS_WRITE
6931	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6932	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6933	{
6934	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6935	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6936	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6937	return VINF_HM_PENDING_XCPT;
6938	}
6939
6940	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6941	if ( GCPtrFirst32 > pSel->u32Limit
6942	\|\| GCPtrLast32 > pSel->u32Limit)
6943	{
6944	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6945	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6946	if (iSegReg == X86_SREG_SS)
6947	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6948	else
6949	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6950	return VINF_HM_PENDING_XCPT;
6951	}
6952	}
6953	}
6954	else
6955	{
6956	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6957	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6958	return VINF_HM_PENDING_XCPT;
6959	}
6960
6961	*pGCPtrMem = GCPtrMem;
6962	return VINF_SUCCESS;
6963	}
6964
6965
6966	/**
6967	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6968	* guest attempting to execute a VMX instruction.
6969	*
6970	* @returns Strict VBox status code (i.e. informational status codes too).
6971	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6972	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6973	*
6974	* @param pVCpu The cross context virtual CPU structure.
6975	* @param uExitReason The VM-exit reason.
6976	*
6977	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6978	* @remarks No-long-jump zone!!!
6979	*/
6980	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6981	{
6982	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6983	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6984
6985	/*
6986	* The physical CPU would have already checked the CPU mode/code segment.
6987	* We shall just assert here for paranoia.
6988	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
6989	*/
6990	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6991	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6992	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
6993
6994	if (uExitReason == VMX_EXIT_VMXON)
6995	{
6996	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6997
6998	/*
6999	* We check CR4.VMXE because it is required to be always set while in VMX operation
7000	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
7001	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7002	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7003	*/
7004	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7005	{
7006	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7007	hmR0VmxSetPendingXcptUD(pVCpu);
7008	return VINF_HM_PENDING_XCPT;
7009	}
7010	}
7011	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7012	{
7013	/*
7014	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7015	* (other than VMXON), we need to raise a #UD.
7016	*/
7017	Log4Func(("Not in VMX root mode -> #UD\n"));
7018	hmR0VmxSetPendingXcptUD(pVCpu);
7019	return VINF_HM_PENDING_XCPT;
7020	}
7021
7022	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7023	return VINF_SUCCESS;
7024	}
7025	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7026
7027
7028	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7029	{
7030	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7031
7032	/*
7033	* If VT-x marks the segment as unusable, most other bits remain undefined:
7034	* - For CS the L, D and G bits have meaning.
7035	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7036	* - For the remaining data segments no bits are defined.
7037	*
7038	* The present bit and the unusable bit has been observed to be set at the
7039	* same time (the selector was supposed to be invalid as we started executing
7040	* a V8086 interrupt in ring-0).
7041	*
7042	* What should be important for the rest of the VBox code, is that the P bit is
7043	* cleared. Some of the other VBox code recognizes the unusable bit, but
7044	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7045	* safe side here, we'll strip off P and other bits we don't care about. If
7046	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7047	*
7048	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7049	*/
7050	#ifdef VBOX_STRICT
7051	uint32_t const uAttr = pSelReg->Attr.u;
7052	#endif
7053
7054	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7055	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7056	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7057
7058	#ifdef VBOX_STRICT
7059	VMMRZCallRing3Disable(pVCpu);
7060	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7061	# ifdef DEBUG_bird
7062	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7063	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7064	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7065	# endif
7066	VMMRZCallRing3Enable(pVCpu);
7067	NOREF(uAttr);
7068	#endif
7069	RT_NOREF2(pVCpu, idxSel);
7070	}
7071
7072
7073	/**
7074	* Imports a guest segment register from the current VMCS into the guest-CPU
7075	* context.
7076	*
7077	* @returns VBox status code.
7078	* @param pVCpu The cross context virtual CPU structure.
7079	* @param iSegReg The segment register number (X86_SREG_XXX).
7080	*
7081	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7082	* do not log!
7083	*/
7084	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7085	{
7086	Assert(iSegReg < X86_SREG_COUNT);
7087
7088	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7089	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7090	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7091	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7092	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7093	#else
7094	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7095	#endif
7096	uint64_t u64Base;
7097	uint32_t u32Sel, u32Limit, u32Attr;
7098	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7099	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7100	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7101	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7102	if (RT_SUCCESS(rc))
7103	{
7104	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7105	pSelReg->Sel = u32Sel;
7106	pSelReg->ValidSel = u32Sel;
7107	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7108	pSelReg->u32Limit = u32Limit;
7109	pSelReg->u64Base = u64Base;
7110	pSelReg->Attr.u = u32Attr;
7111	if (u32Attr & X86DESCATTR_UNUSABLE)
7112	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7113	}
7114	return rc;
7115	}
7116
7117
7118	/**
7119	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7120	*
7121	* @returns VBox status code.
7122	* @param pVCpu The cross context virtual CPU structure.
7123	*
7124	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7125	* do not log!
7126	*/
7127	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7128	{
7129	uint64_t u64Base;
7130	uint32_t u32Sel, u32Limit, u32Attr;
7131	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7132	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7133	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7134	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7135	if (RT_SUCCESS(rc))
7136	{
7137	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7138	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7139	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7140	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7141	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7142	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7143	if (u32Attr & X86DESCATTR_UNUSABLE)
7144	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7145	}
7146	return rc;
7147	}
7148
7149
7150	/**
7151	* Imports the guest TR from the current VMCS into the guest-CPU context.
7152	*
7153	* @returns VBox status code.
7154	* @param pVCpu The cross context virtual CPU structure.
7155	*
7156	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7157	* do not log!
7158	*/
7159	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7160	{
7161	uint32_t u32Sel, u32Limit, u32Attr;
7162	uint64_t u64Base;
7163	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7164	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7165	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7166	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7167	AssertRCReturn(rc, rc);
7168
7169	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7170	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7171	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7172	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7173	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7174	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7175	/* TR is the only selector that can never be unusable. */
7176	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7177	return VINF_SUCCESS;
7178	}
7179
7180
7181	/**
7182	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7183	*
7184	* @returns VBox status code.
7185	* @param pVCpu The cross context virtual CPU structure.
7186	*
7187	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7188	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7189	* instead!!!
7190	*/
7191	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7192	{
7193	uint64_t u64Val;
7194	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7195	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7196	{
7197	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7198	if (RT_SUCCESS(rc))
7199	{
7200	pCtx->rip = u64Val;
7201	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7202	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7203	}
7204	return rc;
7205	}
7206	return VINF_SUCCESS;
7207	}
7208
7209
7210	/**
7211	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7212	*
7213	* @returns VBox status code.
7214	* @param pVCpu The cross context virtual CPU structure.
7215	* @param pVmcsInfo The VMCS info. object.
7216	*
7217	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7218	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7219	* instead!!!
7220	*/
7221	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7222	{
7223	uint32_t u32Val;
7224	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7225	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7226	{
7227	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7228	if (RT_SUCCESS(rc))
7229	{
7230	pCtx->eflags.u32 = u32Val;
7231
7232	/* Restore eflags for real-on-v86-mode hack. */
7233	if (pVmcsInfo->RealMode.fRealOnV86Active)
7234	{
7235	pCtx->eflags.Bits.u1VM = 0;
7236	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7237	}
7238	}
7239	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7240	return rc;
7241	}
7242	return VINF_SUCCESS;
7243	}
7244
7245
7246	/**
7247	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7248	* context.
7249	*
7250	* @returns VBox status code.
7251	* @param pVCpu The cross context virtual CPU structure.
7252	* @param pVmcsInfo The VMCS info. object.
7253	*
7254	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7255	* do not log!
7256	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7257	* instead!!!
7258	*/
7259	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7260	{
7261	uint32_t u32Val;
7262	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7263	if (RT_SUCCESS(rc))
7264	{
7265	if (!u32Val)
7266	{
7267	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7268	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7269
7270	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7271	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7272	}
7273	else
7274	{
7275	/*
7276	* We must import RIP here to set our EM interrupt-inhibited state.
7277	* We also import RFLAGS as our code that evaluates pending interrupts
7278	* before VM-entry requires it.
7279	*/
7280	rc = hmR0VmxImportGuestRip(pVCpu);
7281	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7282	if (RT_SUCCESS(rc))
7283	{
7284	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7285	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7286	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7287	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7288
7289	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7290	{
7291	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7292	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7293	}
7294	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7295	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7296	}
7297	}
7298	}
7299	return rc;
7300	}
7301
7302
7303	/**
7304	* Worker for VMXR0ImportStateOnDemand.
7305	*
7306	* @returns VBox status code.
7307	* @param pVCpu The cross context virtual CPU structure.
7308	* @param pVmcsInfo The VMCS info. object.
7309	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7310	*/
7311	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7312	{
7313	#define VMXLOCAL_BREAK_RC(a_rc) \
7314	if (RT_SUCCESS(a_rc)) \
7315	{ } \
7316	else \
7317	break
7318
7319	int rc = VINF_SUCCESS;
7320	PVM pVM = pVCpu->CTX_SUFF(pVM);
7321	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7322	uint64_t u64Val;
7323	uint32_t u32Val;
7324
7325	/*
7326	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7327	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7328	* neither are other host platforms.
7329	*
7330	* Committing this temporarily as it prevents BSOD.
7331	*
7332	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7333	*/
7334	#ifdef RT_OS_WINDOWS
7335	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7336	return VERR_HM_IPE_1;
7337	#endif
7338
7339	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7340
7341	/*
7342	* We disable interrupts to make the updating of the state and in particular
7343	* the fExtrn modification atomic wrt to preemption hooks.
7344	*/
7345	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7346
7347	fWhat &= pCtx->fExtrn;
7348	if (fWhat)
7349	{
7350	do
7351	{
7352	if (fWhat & CPUMCTX_EXTRN_RIP)
7353	{
7354	rc = hmR0VmxImportGuestRip(pVCpu);
7355	VMXLOCAL_BREAK_RC(rc);
7356	}
7357
7358	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7359	{
7360	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7361	VMXLOCAL_BREAK_RC(rc);
7362	}
7363
7364	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7365	{
7366	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7367	VMXLOCAL_BREAK_RC(rc);
7368	}
7369
7370	if (fWhat & CPUMCTX_EXTRN_RSP)
7371	{
7372	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7373	VMXLOCAL_BREAK_RC(rc);
7374	pCtx->rsp = u64Val;
7375	}
7376
7377	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7378	{
7379	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7380	if (fWhat & CPUMCTX_EXTRN_CS)
7381	{
7382	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7383	rc \|= hmR0VmxImportGuestRip(pVCpu);
7384	if (fRealOnV86Active)
7385	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7386	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7387	}
7388	if (fWhat & CPUMCTX_EXTRN_SS)
7389	{
7390	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7391	if (fRealOnV86Active)
7392	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7393	}
7394	if (fWhat & CPUMCTX_EXTRN_DS)
7395	{
7396	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7397	if (fRealOnV86Active)
7398	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7399	}
7400	if (fWhat & CPUMCTX_EXTRN_ES)
7401	{
7402	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7403	if (fRealOnV86Active)
7404	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7405	}
7406	if (fWhat & CPUMCTX_EXTRN_FS)
7407	{
7408	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7409	if (fRealOnV86Active)
7410	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7411	}
7412	if (fWhat & CPUMCTX_EXTRN_GS)
7413	{
7414	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7415	if (fRealOnV86Active)
7416	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7417	}
7418	VMXLOCAL_BREAK_RC(rc);
7419	}
7420
7421	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7422	{
7423	if (fWhat & CPUMCTX_EXTRN_LDTR)
7424	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7425
7426	if (fWhat & CPUMCTX_EXTRN_GDTR)
7427	{
7428	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7429	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7430	pCtx->gdtr.pGdt = u64Val;
7431	pCtx->gdtr.cbGdt = u32Val;
7432	}
7433
7434	/* Guest IDTR. */
7435	if (fWhat & CPUMCTX_EXTRN_IDTR)
7436	{
7437	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7438	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7439	pCtx->idtr.pIdt = u64Val;
7440	pCtx->idtr.cbIdt = u32Val;
7441	}
7442
7443	/* Guest TR. */
7444	if (fWhat & CPUMCTX_EXTRN_TR)
7445	{
7446	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7447	don't need to import that one. */
7448	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7449	rc \|= hmR0VmxImportGuestTr(pVCpu);
7450	}
7451	VMXLOCAL_BREAK_RC(rc);
7452	}
7453
7454	if (fWhat & CPUMCTX_EXTRN_DR7)
7455	{
7456	if (!pVCpu->hm.s.fUsingHyperDR7)
7457	{
7458	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7459	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7460	VMXLOCAL_BREAK_RC(rc);
7461	pCtx->dr[7] = u32Val;
7462	}
7463	}
7464
7465	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7466	{
7467	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7468	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7469	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7470	pCtx->SysEnter.cs = u32Val;
7471	VMXLOCAL_BREAK_RC(rc);
7472	}
7473
7474	#if HC_ARCH_BITS == 64
7475	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7476	{
7477	if ( pVM->hm.s.fAllow64BitGuests
7478	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7479	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7480	}
7481
7482	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7483	{
7484	if ( pVM->hm.s.fAllow64BitGuests
7485	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7486	{
7487	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7488	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7489	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7490	}
7491	}
7492	#endif
7493
7494	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7495	#if HC_ARCH_BITS == 32
7496	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7497	#endif
7498	)
7499	{
7500	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7501	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7502	Assert(pMsrs);
7503	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7504	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7505	for (uint32_t i = 0; i < cMsrs; i++)
7506	{
7507	uint32_t const idMsr = pMsrs[i].u32Msr;
7508	switch (idMsr)
7509	{
7510	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7511	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7512	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7513	#if HC_ARCH_BITS == 32
7514	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7515	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7516	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7517	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7518	#endif
7519	default:
7520	{
7521	pCtx->fExtrn = 0;
7522	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7523	ASMSetFlags(fEFlags);
7524	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7525	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7526	}
7527	}
7528	}
7529	}
7530
7531	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7532	{
7533	uint64_t u64Shadow;
7534	if (fWhat & CPUMCTX_EXTRN_CR0)
7535	{
7536	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7537	* remove when we drop 32-bit host w/ 64-bit host support, see
7538	* @bugref{9180#c39}. */
7539	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7540	#if HC_ARCH_BITS == 32
7541	uint32_t u32Shadow;
7542	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7543	u64Shadow = u32Shadow;
7544	#else
7545	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7546	#endif
7547	VMXLOCAL_BREAK_RC(rc);
7548	u64Val = u32Val;
7549	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7550	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7551	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7552	CPUMSetGuestCR0(pVCpu, u64Val);
7553	VMMRZCallRing3Enable(pVCpu);
7554	}
7555
7556	if (fWhat & CPUMCTX_EXTRN_CR4)
7557	{
7558	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7559	* remove when we drop 32-bit host w/ 64-bit host support, see
7560	* @bugref{9180#c39}. */
7561	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7562	#if HC_ARCH_BITS == 32
7563	uint32_t u32Shadow;
7564	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7565	u64Shadow = u32Shadow;
7566	#else
7567	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7568	#endif
7569	VMXLOCAL_BREAK_RC(rc);
7570	u64Val = u32Val;
7571	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7572	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7573	pCtx->cr4 = u64Val;
7574	}
7575
7576	if (fWhat & CPUMCTX_EXTRN_CR3)
7577	{
7578	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7579	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7580	\|\| ( pVM->hm.s.fNestedPaging
7581	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7582	{
7583	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7584	VMXLOCAL_BREAK_RC(rc);
7585	if (pCtx->cr3 != u64Val)
7586	{
7587	pCtx->cr3 = u64Val;
7588	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7589	}
7590
7591	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7592	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7593	if (CPUMIsGuestInPAEModeEx(pCtx))
7594	{
7595	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7596	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7597	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7598	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7599	VMXLOCAL_BREAK_RC(rc);
7600	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7601	}
7602	}
7603	}
7604
7605	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7606	# if 0
7607	/** @todo NSTVMX: We handle each of these fields individually by passing it to IEM
7608	* VM-exit handlers. We might handle it differently when using the fast path. */
7609	/*
7610	* The hardware virtualization state currently consists of VMCS fields that may be
7611	* modified by execution of the nested-guest (that are not part of the general
7612	* guest state) and is visible to guest software. Hence, it is technically part of
7613	* the guest-CPU state when executing a nested-guest.
7614	*/
7615	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7616	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7617	{
7618	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7619	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7620	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7621	VMXLOCAL_BREAK_RC(rc);
7622
7623	/*
7624	* VM-entry can fail due to invalid-guest state, machine-check events and
7625	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7626	* all other VMCS fields are left unmodified on VM-entry failure.
7627	*
7628	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7629	*/
7630	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7631	if (!fEntryFailed)
7632	{
7633	/*
7634	* Some notes on VMCS fields that may need importing when the fast path
7635	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7636	*
7637	* Requires fixing up when using hardware-assisted VMX:
7638	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7639	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7640	* - IDT-vectoring info: Think about this.
7641	* - IDT-vectoring error code: Think about this.
7642	*
7643	* Emulated:
7644	* - Guest-interruptiblity state: Derived from FFs and RIP.
7645	* - Guest pending debug exceptions: Derived from DR6.
7646	* - Guest activity state: Emulated from EM state.
7647	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7648	* - Entry-interrupt info: Emulated, cleared to 0.
7649	*/
7650	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7651	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7652	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7653	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7654	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7655	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7656	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7657	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7658	/** @todo NSTVMX: Save and adjust preemption timer value. */
7659	}
7660
7661	VMXLOCAL_BREAK_RC(rc);
7662	}
7663	# endif
7664	#endif
7665	}
7666	} while (0);
7667
7668	if (RT_SUCCESS(rc))
7669	{
7670	/* Update fExtrn. */
7671	pCtx->fExtrn &= ~fWhat;
7672
7673	/* If everything has been imported, clear the HM keeper bit. */
7674	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7675	{
7676	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7677	Assert(!pCtx->fExtrn);
7678	}
7679	}
7680	}
7681	else
7682	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7683
7684	ASMSetFlags(fEFlags);
7685
7686	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7687
7688	if (RT_SUCCESS(rc))
7689	{ /* likely */ }
7690	else
7691	return rc;
7692
7693	/*
7694	* Honor any pending CR3 updates.
7695	*
7696	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7697	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7698	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7699	*
7700	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7701	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7702	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7703	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7704	*
7705	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7706	*/
7707	if (VMMRZCallRing3IsEnabled(pVCpu))
7708	{
7709	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7710	{
7711	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7712	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7713	}
7714
7715	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7716	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7717
7718	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7719	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7720	}
7721
7722	return VINF_SUCCESS;
7723	#undef VMXLOCAL_BREAK_RC
7724	}
7725
7726
7727	/**
7728	* Saves the guest state from the VMCS into the guest-CPU context.
7729	*
7730	* @returns VBox status code.
7731	* @param pVCpu The cross context virtual CPU structure.
7732	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7733	*/
7734	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7735	{
7736	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7737	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7738	}
7739
7740
7741	/**
7742	* Check per-VM and per-VCPU force flag actions that require us to go back to
7743	* ring-3 for one reason or another.
7744	*
7745	* @returns Strict VBox status code (i.e. informational status codes too)
7746	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7747	* ring-3.
7748	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7749	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7750	* interrupts)
7751	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7752	* all EMTs to be in ring-3.
7753	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7754	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7755	* to the EM loop.
7756	*
7757	* @param pVCpu The cross context virtual CPU structure.
7758	* @param fStepping Whether we are single-stepping the guest using the
7759	* hypervisor debugger.
7760	*/
7761	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7762	{
7763	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7764
7765	/*
7766	* Update pending interrupts into the APIC's IRR.
7767	*/
7768	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7769	APICUpdatePendingInterrupts(pVCpu);
7770
7771	/*
7772	* Anything pending? Should be more likely than not if we're doing a good job.
7773	*/
7774	PVM pVM = pVCpu->CTX_SUFF(pVM);
7775	if ( !fStepping
7776	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7777	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7778	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7779	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7780	return VINF_SUCCESS;
7781
7782	/* Pending PGM C3 sync. */
7783	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7784	{
7785	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7786	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7787	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7788	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7789	if (rcStrict2 != VINF_SUCCESS)
7790	{
7791	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7792	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7793	return rcStrict2;
7794	}
7795	}
7796
7797	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7798	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7799	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7800	{
7801	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7802	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7803	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7804	return rc2;
7805	}
7806
7807	/* Pending VM request packets, such as hardware interrupts. */
7808	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7809	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7810	{
7811	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7812	return VINF_EM_PENDING_REQUEST;
7813	}
7814
7815	/* Pending PGM pool flushes. */
7816	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7817	{
7818	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7819	return VINF_PGM_POOL_FLUSH_PENDING;
7820	}
7821
7822	/* Pending DMA requests. */
7823	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7824	{
7825	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7826	return VINF_EM_RAW_TO_R3;
7827	}
7828
7829	return VINF_SUCCESS;
7830	}
7831
7832
7833	/**
7834	* Converts any TRPM trap into a pending HM event. This is typically used when
7835	* entering from ring-3 (not longjmp returns).
7836	*
7837	* @param pVCpu The cross context virtual CPU structure.
7838	*/
7839	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7840	{
7841	Assert(TRPMHasTrap(pVCpu));
7842	Assert(!pVCpu->hm.s.Event.fPending);
7843
7844	uint8_t uVector;
7845	TRPMEVENT enmTrpmEvent;
7846	RTGCUINT uErrCode;
7847	RTGCUINTPTR GCPtrFaultAddress;
7848	uint8_t cbInstr;
7849
7850	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7851	AssertRC(rc);
7852
7853	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7854	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7855	if (enmTrpmEvent == TRPM_TRAP)
7856	{
7857	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7858	* generated using INT1 (ICEBP). */
7859	switch (uVector)
7860	{
7861	case X86_XCPT_NMI:
7862	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7863	break;
7864
7865	case X86_XCPT_BP:
7866	case X86_XCPT_OF:
7867	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7868	break;
7869
7870	case X86_XCPT_PF:
7871	case X86_XCPT_DF:
7872	case X86_XCPT_TS:
7873	case X86_XCPT_NP:
7874	case X86_XCPT_SS:
7875	case X86_XCPT_GP:
7876	case X86_XCPT_AC:
7877	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7878	RT_FALL_THRU();
7879	default:
7880	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7881	break;
7882	}
7883	}
7884	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7885	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7886	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7887	{
7888	switch (uVector)
7889	{
7890	case X86_XCPT_BP:
7891	case X86_XCPT_OF:
7892	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7893	break;
7894
7895	default:
7896	Assert(uVector == X86_XCPT_DB);
7897	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7898	break;
7899	}
7900	}
7901	else
7902	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7903
7904	rc = TRPMResetTrap(pVCpu);
7905	AssertRC(rc);
7906	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7907	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7908
7909	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7910	}
7911
7912
7913	/**
7914	* Converts the pending HM event into a TRPM trap.
7915	*
7916	* @param pVCpu The cross context virtual CPU structure.
7917	*/
7918	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7919	{
7920	Assert(pVCpu->hm.s.Event.fPending);
7921
7922	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7923	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7924	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7925	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7926
7927	/* If a trap was already pending, we did something wrong! */
7928	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7929
7930	/** @todo Use HMVmxEventToTrpmEventType() later. */
7931	TRPMEVENT enmTrapType;
7932	switch (uVectorType)
7933	{
7934	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7935	enmTrapType = TRPM_HARDWARE_INT;
7936	break;
7937
7938	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7939	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7940	enmTrapType = TRPM_TRAP;
7941	break;
7942
7943	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7944	Assert(uVector == X86_XCPT_DB);
7945	enmTrapType = TRPM_SOFTWARE_INT;
7946	break;
7947
7948	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7949	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7950	enmTrapType = TRPM_SOFTWARE_INT;
7951	break;
7952
7953	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7954	enmTrapType = TRPM_SOFTWARE_INT;
7955	break;
7956
7957	default:
7958	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7959	enmTrapType = TRPM_32BIT_HACK;
7960	break;
7961	}
7962
7963	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7964
7965	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7966	AssertRC(rc);
7967
7968	if (fErrorCodeValid)
7969	TRPMSetErrorCode(pVCpu, uErrorCode);
7970
7971	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7972	&& uVector == X86_XCPT_PF)
7973	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7974	else if (enmTrapType == TRPM_SOFTWARE_INT)
7975	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7976
7977	/* We're now done converting the pending event. */
7978	pVCpu->hm.s.Event.fPending = false;
7979	}
7980
7981
7982	/**
7983	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7984	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7985	*
7986	* @param pVCpu The cross context virtual CPU structure.
7987	* @param pVmcsInfo The VMCS info. object.
7988	*/
7989	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7990	{
7991	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7992	{
7993	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7994	{
7995	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7996	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7997	AssertRC(rc);
7998	}
7999	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8000	}
8001
8002
8003	/**
8004	* Clears the interrupt-window exiting control in the VMCS.
8005	*
8006	* @param pVmcsInfo The VMCS info. object.
8007	*/
8008	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8009	{
8010	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8011	{
8012	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8013	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8014	}
8015	return VINF_SUCCESS;
8016	}
8017
8018
8019	/**
8020	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8021	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8022	*
8023	* @param pVCpu The cross context virtual CPU structure.
8024	* @param pVmcsInfo The VMCS info. object.
8025	*/
8026	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8027	{
8028	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8029	{
8030	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8031	{
8032	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8033	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8034	AssertRC(rc);
8035	Log4Func(("Setup NMI-window exiting\n"));
8036	}
8037	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8038	}
8039
8040
8041	/**
8042	* Clears the NMI-window exiting control in the VMCS.
8043	*
8044	* @param pVmcsInfo The VMCS info. object.
8045	*/
8046	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8047	{
8048	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8049	{
8050	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8051	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8052	}
8053	return VINF_SUCCESS;
8054	}
8055
8056
8057	/**
8058	* Does the necessary state syncing before returning to ring-3 for any reason
8059	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8060	*
8061	* @returns VBox status code.
8062	* @param pVCpu The cross context virtual CPU structure.
8063	* @param fImportState Whether to import the guest state from the VMCS back
8064	* to the guest-CPU context.
8065	*
8066	* @remarks No-long-jmp zone!!!
8067	*/
8068	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8069	{
8070	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8071	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8072
8073	RTCPUID idCpu = RTMpCpuId();
8074	Log4Func(("HostCpuId=%u\n", idCpu));
8075
8076	/*
8077	* !!! IMPORTANT !!!
8078	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8079	*/
8080
8081	/* Save the guest state if necessary. */
8082	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8083	if (fImportState)
8084	{
8085	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8086	AssertRCReturn(rc, rc);
8087	}
8088
8089	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8090	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8091	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8092
8093	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8094	#ifdef VBOX_STRICT
8095	if (CPUMIsHyperDebugStateActive(pVCpu))
8096	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8097	#endif
8098	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8099	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8100	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8101
8102	#if HC_ARCH_BITS == 64
8103	/* Restore host-state bits that VT-x only restores partially. */
8104	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8105	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8106	{
8107	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8108	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8109	}
8110	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8111	#endif
8112
8113	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8114	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8115	{
8116	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8117	if (!fImportState)
8118	{
8119	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8120	AssertRCReturn(rc, rc);
8121	}
8122	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8123	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8124	}
8125	else
8126	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8127
8128	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8129	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8130
8131	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8132	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8133	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8134	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8135	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8136	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8137	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8138	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8139	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8140
8141	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8142
8143	/** @todo This partially defeats the purpose of having preemption hooks.
8144	* The problem is, deregistering the hooks should be moved to a place that
8145	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8146	* context.
8147	*/
8148	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8149	AssertRCReturn(rc, rc);
8150
8151	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8152	NOREF(idCpu);
8153	return VINF_SUCCESS;
8154	}
8155
8156
8157	/**
8158	* Leaves the VT-x session.
8159	*
8160	* @returns VBox status code.
8161	* @param pVCpu The cross context virtual CPU structure.
8162	*
8163	* @remarks No-long-jmp zone!!!
8164	*/
8165	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8166	{
8167	HM_DISABLE_PREEMPT(pVCpu);
8168	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8169	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8170	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8171
8172	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8173	and done this from the VMXR0ThreadCtxCallback(). */
8174	if (!pVCpu->hm.s.fLeaveDone)
8175	{
8176	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8177	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8178	pVCpu->hm.s.fLeaveDone = true;
8179	}
8180	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8181
8182	/*
8183	* !!! IMPORTANT !!!
8184	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8185	*/
8186
8187	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8188	/** @todo Deregistering here means we need to VMCLEAR always
8189	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8190	* for calling VMMR0ThreadCtxHookDisable here! */
8191	VMMR0ThreadCtxHookDisable(pVCpu);
8192
8193	/* Leave HM context. This takes care of local init (term). */
8194	int rc = HMR0LeaveCpu(pVCpu);
8195
8196	HM_RESTORE_PREEMPT();
8197	return rc;
8198	}
8199
8200
8201	/**
8202	* Does the necessary state syncing before doing a longjmp to ring-3.
8203	*
8204	* @returns VBox status code.
8205	* @param pVCpu The cross context virtual CPU structure.
8206	*
8207	* @remarks No-long-jmp zone!!!
8208	*/
8209	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8210	{
8211	return hmR0VmxLeaveSession(pVCpu);
8212	}
8213
8214
8215	/**
8216	* Take necessary actions before going back to ring-3.
8217	*
8218	* An action requires us to go back to ring-3. This function does the necessary
8219	* steps before we can safely return to ring-3. This is not the same as longjmps
8220	* to ring-3, this is voluntary and prepares the guest so it may continue
8221	* executing outside HM (recompiler/IEM).
8222	*
8223	* @returns VBox status code.
8224	* @param pVCpu The cross context virtual CPU structure.
8225	* @param rcExit The reason for exiting to ring-3. Can be
8226	* VINF_VMM_UNKNOWN_RING3_CALL.
8227	*/
8228	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8229	{
8230	Assert(pVCpu);
8231	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8232
8233	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8234	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8235	{
8236	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8237	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8238	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8239	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8240	}
8241
8242	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8243	VMMRZCallRing3Disable(pVCpu);
8244	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8245
8246	/*
8247	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8248	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8249	*
8250	* This is because execution may continue from ring-3 and we would need to inject
8251	* the event from there (hence place it back in TRPM).
8252	*/
8253	if (pVCpu->hm.s.Event.fPending)
8254	{
8255	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8256	Assert(!pVCpu->hm.s.Event.fPending);
8257
8258	/* Clear the events from the VMCS. */
8259	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8260	AssertRCReturn(rc, rc);
8261	}
8262	#ifdef VBOX_STRICT
8263	else
8264	{
8265	/*
8266	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8267	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8268	* occasionally, see @bugref{9180#c42}.
8269	*/
8270	uint32_t uEntryIntInfo;
8271	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8272	AssertRC(rc);
8273	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8274	}
8275	#endif
8276
8277	/*
8278	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8279	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8280	* (e.g. TPR below threshold).
8281	*/
8282	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8283	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8284	AssertRCReturn(rc, rc);
8285
8286	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8287	and if we're injecting an event we should have a TRPM trap pending. */
8288	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8289	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8290	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8291	#endif
8292
8293	/* Save guest state and restore host state bits. */
8294	rc = hmR0VmxLeaveSession(pVCpu);
8295	AssertRCReturn(rc, rc);
8296	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8297
8298	/* Thread-context hooks are unregistered at this point!!! */
8299
8300	/* Sync recompiler state. */
8301	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8302	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8303	\| CPUM_CHANGED_LDTR
8304	\| CPUM_CHANGED_GDTR
8305	\| CPUM_CHANGED_IDTR
8306	\| CPUM_CHANGED_TR
8307	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8308	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8309	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8310	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8311
8312	Assert(!pVCpu->hm.s.fClearTrapFlag);
8313
8314	/* Update the exit-to-ring 3 reason. */
8315	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8316
8317	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8318	if ( rcExit != VINF_EM_RAW_INTERRUPT
8319	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8320	{
8321	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8322	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8323	}
8324
8325	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8326
8327	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8328	VMMRZCallRing3RemoveNotification(pVCpu);
8329	VMMRZCallRing3Enable(pVCpu);
8330
8331	return rc;
8332	}
8333
8334
8335	/**
8336	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8337	* longjump to ring-3 and possibly get preempted.
8338	*
8339	* @returns VBox status code.
8340	* @param pVCpu The cross context virtual CPU structure.
8341	* @param enmOperation The operation causing the ring-3 longjump.
8342	* @param pvUser User argument, currently unused, NULL.
8343	*/
8344	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8345	{
8346	RT_NOREF(pvUser);
8347	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8348	{
8349	/*
8350	* !!! IMPORTANT !!!
8351	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8352	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8353	*/
8354	VMMRZCallRing3RemoveNotification(pVCpu);
8355	VMMRZCallRing3Disable(pVCpu);
8356	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8357	RTThreadPreemptDisable(&PreemptState);
8358
8359	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8360	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8361	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8362	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8363
8364	#if HC_ARCH_BITS == 64
8365	/* Restore host-state bits that VT-x only restores partially. */
8366	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8367	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8368	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8369	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8370	#endif
8371
8372	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8373	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8374	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8375
8376	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8377	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8378	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8379
8380	/* Clear the current VMCS data back to memory. */
8381	hmR0VmxClearVmcs(pVmcsInfo);
8382
8383	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8384	VMMR0ThreadCtxHookDisable(pVCpu);
8385	HMR0LeaveCpu(pVCpu);
8386	RTThreadPreemptRestore(&PreemptState);
8387	return VINF_SUCCESS;
8388	}
8389
8390	Assert(pVCpu);
8391	Assert(pvUser);
8392	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8393	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8394
8395	VMMRZCallRing3Disable(pVCpu);
8396	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8397
8398	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8399
8400	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8401	AssertRCReturn(rc, rc);
8402
8403	VMMRZCallRing3Enable(pVCpu);
8404	return VINF_SUCCESS;
8405	}
8406
8407
8408	/**
8409	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8410	* stack.
8411	*
8412	* @returns Strict VBox status code (i.e. informational status codes too).
8413	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8414	* @param pVCpu The cross context virtual CPU structure.
8415	* @param uValue The value to push to the guest stack.
8416	*/
8417	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8418	{
8419	/*
8420	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8421	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8422	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8423	*/
8424	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8425	if (pCtx->sp == 1)
8426	return VINF_EM_RESET;
8427	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8428	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8429	AssertRC(rc);
8430	return rc;
8431	}
8432
8433
8434	/**
8435	* Injects an event into the guest upon VM-entry by updating the relevant fields
8436	* in the VM-entry area in the VMCS.
8437	*
8438	* @returns Strict VBox status code (i.e. informational status codes too).
8439	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8440	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8441	*
8442	* @param pVCpu The cross context virtual CPU structure.
8443	* @param pVmxTransient The VMX-transient structure.
8444	* @param pEvent The event being injected.
8445	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8446	* This will be updated if necessary. This cannot not
8447	* be NULL.
8448	* @param fStepping Whether we're single-stepping guest execution and
8449	* should return VINF_EM_DBG_STEPPED if the event is
8450	* injected directly (registers modified by us, not by
8451	* hardware on VM-entry).
8452	*/
8453	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8454	uint32_t *pfIntrState)
8455	{
8456	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8457	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8458	Assert(pfIntrState);
8459
8460	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8461	uint32_t u32IntInfo = pEvent->u64IntInfo;
8462	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8463	uint32_t const cbInstr = pEvent->cbInstr;
8464	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8465	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8466	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8467
8468	#ifdef VBOX_STRICT
8469	/*
8470	* Validate the error-code-valid bit for hardware exceptions.
8471	* No error codes for exceptions in real-mode.
8472	*
8473	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8474	*/
8475	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8476	&& !CPUMIsGuestInRealModeEx(pCtx))
8477	{
8478	switch (uVector)
8479	{
8480	case X86_XCPT_PF:
8481	case X86_XCPT_DF:
8482	case X86_XCPT_TS:
8483	case X86_XCPT_NP:
8484	case X86_XCPT_SS:
8485	case X86_XCPT_GP:
8486	case X86_XCPT_AC:
8487	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8488	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8489	RT_FALL_THRU();
8490	default:
8491	break;
8492	}
8493	}
8494
8495	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8496	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8497	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8498	#endif
8499
8500	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8501
8502	/*
8503	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8504	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8505	* interrupt handler in the (real-mode) guest.
8506	*
8507	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8508	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8509	*/
8510	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8511	{
8512	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8513	{
8514	/*
8515	* For CPUs with unrestricted guest execution enabled and with the guest
8516	* in real-mode, we must not set the deliver-error-code bit.
8517	*
8518	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8519	*/
8520	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8521	}
8522	else
8523	{
8524	PVM pVM = pVCpu->CTX_SUFF(pVM);
8525	Assert(PDMVmmDevHeapIsEnabled(pVM));
8526	Assert(pVM->hm.s.vmx.pRealModeTSS);
8527	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8528
8529	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8530	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8531	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8532	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8533	AssertRCReturn(rc2, rc2);
8534
8535	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8536	size_t const cbIdtEntry = sizeof(X86IDTR16);
8537	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8538	{
8539	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8540	if (uVector == X86_XCPT_DF)
8541	return VINF_EM_RESET;
8542
8543	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8544	No error codes for exceptions in real-mode. */
8545	if (uVector == X86_XCPT_GP)
8546	{
8547	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8548	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8549	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8550	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8551	HMEVENT EventXcptDf;
8552	RT_ZERO(EventXcptDf);
8553	EventXcptDf.u64IntInfo = uXcptDfInfo;
8554	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8555	}
8556
8557	/*
8558	* If we're injecting an event with no valid IDT entry, inject a #GP.
8559	* No error codes for exceptions in real-mode.
8560	*
8561	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8562	*/
8563	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8564	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8565	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8566	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8567	HMEVENT EventXcptGp;
8568	RT_ZERO(EventXcptGp);
8569	EventXcptGp.u64IntInfo = uXcptGpInfo;
8570	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8571	}
8572
8573	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8574	uint16_t uGuestIp = pCtx->ip;
8575	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8576	{
8577	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8578	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8579	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8580	}
8581	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8582	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8583
8584	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8585	X86IDTR16 IdtEntry;
8586	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8587	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8588	AssertRCReturn(rc2, rc2);
8589
8590	/* Construct the stack frame for the interrupt/exception handler. */
8591	VBOXSTRICTRC rcStrict;
8592	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8593	if (rcStrict == VINF_SUCCESS)
8594	{
8595	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8596	if (rcStrict == VINF_SUCCESS)
8597	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8598	}
8599
8600	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8601	if (rcStrict == VINF_SUCCESS)
8602	{
8603	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8604	pCtx->rip = IdtEntry.offSel;
8605	pCtx->cs.Sel = IdtEntry.uSel;
8606	pCtx->cs.ValidSel = IdtEntry.uSel;
8607	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8608	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8609	&& uVector == X86_XCPT_PF)
8610	pCtx->cr2 = GCPtrFault;
8611
8612	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8613	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8614	\| HM_CHANGED_GUEST_RSP);
8615
8616	/*
8617	* If we delivered a hardware exception (other than an NMI) and if there was
8618	* block-by-STI in effect, we should clear it.
8619	*/
8620	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8621	{
8622	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8623	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8624	Log4Func(("Clearing inhibition due to STI\n"));
8625	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8626	}
8627
8628	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8629	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8630
8631	/*
8632	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8633	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8634	*/
8635	pVCpu->hm.s.Event.fPending = false;
8636
8637	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8638	if (fStepping)
8639	rcStrict = VINF_EM_DBG_STEPPED;
8640	}
8641	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8642	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8643	return rcStrict;
8644	}
8645	}
8646
8647	/*
8648	* Validate.
8649	*/
8650	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8651	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8652
8653	/*
8654	* Inject the event into the VMCS.
8655	*/
8656	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8657	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8659	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8660	AssertRCReturn(rc, rc);
8661
8662	/*
8663	* Update guest CR2 if this is a page-fault.
8664	*/
8665	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8666	&& uVector == X86_XCPT_PF)
8667	pCtx->cr2 = GCPtrFault;
8668
8669	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8670	return VINF_SUCCESS;
8671	}
8672
8673
8674	/**
8675	* Evaluates the event to be delivered to the guest and sets it as the pending
8676	* event.
8677	*
8678	* @returns Strict VBox status code (i.e. informational status codes too).
8679	* @param pVCpu The cross context virtual CPU structure.
8680	* @param pVmxTransient The VMX-transient structure.
8681	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8682	*/
8683	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8684	{
8685	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8686	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8687
8688	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8689	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8690	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8691	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8692	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8693
8694	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8695	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8696	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8697	Assert(!TRPMHasTrap(pVCpu));
8698	Assert(pfIntrState);
8699
8700	*pfIntrState = fIntrState;
8701
8702	/*
8703	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8704	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8705	*/
8706	/** @todo SMI. SMIs take priority over NMIs. */
8707	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8708	{
8709	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8710	if ( !pVCpu->hm.s.Event.fPending
8711	&& !fBlockNmi
8712	&& !fBlockSti
8713	&& !fBlockMovSS)
8714	{
8715	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8716	if ( pVmxTransient->fIsNestedGuest
8717	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8718	return IEMExecVmxVmexitNmi(pVCpu);
8719	#endif
8720	hmR0VmxSetPendingXcptNmi(pVCpu);
8721	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8722	Log4Func(("Pending NMI\n"));
8723	}
8724	else
8725	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8726	}
8727	/*
8728	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8729	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8730	*/
8731	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8732	&& !pVCpu->hm.s.fSingleInstruction)
8733	{
8734	Assert(!DBGFIsStepping(pVCpu));
8735	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8736	AssertRCReturn(rc, rc);
8737	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8738	if ( !pVCpu->hm.s.Event.fPending
8739	&& !fBlockInt
8740	&& !fBlockSti
8741	&& !fBlockMovSS)
8742	{
8743	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8744	if ( pVmxTransient->fIsNestedGuest
8745	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8746	{
8747	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8748	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8749	return rcStrict;
8750	}
8751	#endif
8752	uint8_t u8Interrupt;
8753	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8754	if (RT_SUCCESS(rc))
8755	{
8756	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8757	if ( pVmxTransient->fIsNestedGuest
8758	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8759	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8760	{
8761	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8762	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8763	return rcStrict;
8764	}
8765	#endif
8766	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8767	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8768	}
8769	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8770	{
8771	if ( !pVmxTransient->fIsNestedGuest
8772	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8773	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8774	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8775
8776	/*
8777	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8778	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8779	* need to re-set this force-flag here.
8780	*/
8781	}
8782	else
8783	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8784	}
8785	else
8786	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8787	}
8788
8789	return VINF_SUCCESS;
8790	}
8791
8792
8793	/**
8794	* Injects any pending events into the guest if the guest is in a state to
8795	* receive them.
8796	*
8797	* @returns Strict VBox status code (i.e. informational status codes too).
8798	* @param pVCpu The cross context virtual CPU structure.
8799	* @param pVmxTransient The VMX-transient structure.
8800	* @param fIntrState The VT-x guest-interruptibility state.
8801	* @param fStepping Whether we are single-stepping the guest using the
8802	* hypervisor debugger and should return
8803	* VINF_EM_DBG_STEPPED if the event was dispatched
8804	* directly.
8805	*/
8806	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8807	{
8808	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8809	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8810
8811	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8812	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8813
8814	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8815	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8816	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8817	Assert(!TRPMHasTrap(pVCpu));
8818
8819	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8820	if (pVCpu->hm.s.Event.fPending)
8821	{
8822	/*
8823	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8824	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8825	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8826	*
8827	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8828	*/
8829	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8830	#ifdef VBOX_STRICT
8831	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8832	{
8833	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8834	Assert(!fBlockInt);
8835	Assert(!fBlockSti);
8836	Assert(!fBlockMovSS);
8837	}
8838	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8839	{
8840	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8841	Assert(!fBlockSti);
8842	Assert(!fBlockMovSS);
8843	Assert(!fBlockNmi);
8844	}
8845	#endif
8846	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8847	uIntType));
8848
8849	/*
8850	* Inject the event and get any changes to the guest-interruptibility state.
8851	*
8852	* The guest-interruptibility state may need to be updated if we inject the event
8853	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8854	*/
8855	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8856	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8857
8858	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8859	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8860	else
8861	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8862	}
8863
8864	/*
8865	* Update the guest-interruptibility state.
8866	*
8867	* This is required for the real-on-v86 software interrupt injection case above, as well as
8868	* updates to the guest state from ring-3 or IEM/REM.
8869	*/
8870	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8871	AssertRCReturn(rc, rc);
8872
8873	/*
8874	* There's no need to clear the VM-entry interruption-information field here if we're not
8875	* injecting anything. VT-x clears the valid bit on every VM-exit.
8876	*
8877	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8878	*/
8879
8880	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8881	NOREF(fBlockMovSS); NOREF(fBlockSti);
8882	return rcStrict;
8883	}
8884
8885
8886	/**
8887	* Enters the VT-x session.
8888	*
8889	* @returns VBox status code.
8890	* @param pVCpu The cross context virtual CPU structure.
8891	*/
8892	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8893	{
8894	AssertPtr(pVCpu);
8895	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8896	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8897
8898	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8899	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8900	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8901
8902	#ifdef VBOX_STRICT
8903	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8904	RTCCUINTREG uHostCR4 = ASMGetCR4();
8905	if (!(uHostCR4 & X86_CR4_VMXE))
8906	{
8907	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8908	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8909	}
8910	#endif
8911
8912	/*
8913	* Load the appropriate VMCS as the current and active one.
8914	*/
8915	PVMXVMCSINFO pVmcsInfo;
8916	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8917	if (!fInNestedGuestMode)
8918	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8919	else
8920	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8921	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8922	if (RT_SUCCESS(rc))
8923	{
8924	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8925	pVCpu->hm.s.fLeaveDone = false;
8926	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8927
8928	/*
8929	* Do the EMT scheduled L1D flush here if needed.
8930	*/
8931	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8932	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8933	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8934	hmR0MdsClear();
8935	}
8936	return rc;
8937	}
8938
8939
8940	/**
8941	* The thread-context callback (only on platforms which support it).
8942	*
8943	* @param enmEvent The thread-context event.
8944	* @param pVCpu The cross context virtual CPU structure.
8945	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8946	* @thread EMT(pVCpu)
8947	*/
8948	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8949	{
8950	NOREF(fGlobalInit);
8951
8952	switch (enmEvent)
8953	{
8954	case RTTHREADCTXEVENT_OUT:
8955	{
8956	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8957	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8958	VMCPU_ASSERT_EMT(pVCpu);
8959
8960	/* No longjmps (logger flushes, locks) in this fragile context. */
8961	VMMRZCallRing3Disable(pVCpu);
8962	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8963
8964	/* Restore host-state (FPU, debug etc.) */
8965	if (!pVCpu->hm.s.fLeaveDone)
8966	{
8967	/*
8968	* Do -not- import the guest-state here as we might already be in the middle of importing
8969	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8970	*/
8971	hmR0VmxLeave(pVCpu, false /* fImportState */);
8972	pVCpu->hm.s.fLeaveDone = true;
8973	}
8974
8975	/* Leave HM context, takes care of local init (term). */
8976	int rc = HMR0LeaveCpu(pVCpu);
8977	AssertRC(rc);
8978
8979	/* Restore longjmp state. */
8980	VMMRZCallRing3Enable(pVCpu);
8981	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8982	break;
8983	}
8984
8985	case RTTHREADCTXEVENT_IN:
8986	{
8987	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8988	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8989	VMCPU_ASSERT_EMT(pVCpu);
8990
8991	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8992	VMMRZCallRing3Disable(pVCpu);
8993	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8994
8995	/* Initialize the bare minimum state required for HM. This takes care of
8996	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8997	int rc = hmR0EnterCpu(pVCpu);
8998	AssertRC(rc);
8999	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9000	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9001
9002	/* Load the active VMCS as the current one. */
9003	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9004	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9005	AssertRC(rc);
9006	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9007	pVCpu->hm.s.fLeaveDone = false;
9008
9009	/* Do the EMT scheduled L1D flush if needed. */
9010	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9011	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9012
9013	/* Restore longjmp state. */
9014	VMMRZCallRing3Enable(pVCpu);
9015	break;
9016	}
9017
9018	default:
9019	break;
9020	}
9021	}
9022
9023
9024	/**
9025	* Exports the host state into the VMCS host-state area.
9026	* Sets up the VM-exit MSR-load area.
9027	*
9028	* The CPU state will be loaded from these fields on every successful VM-exit.
9029	*
9030	* @returns VBox status code.
9031	* @param pVCpu The cross context virtual CPU structure.
9032	*
9033	* @remarks No-long-jump zone!!!
9034	*/
9035	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9036	{
9037	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9038
9039	int rc = VINF_SUCCESS;
9040	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9041	{
9042	rc = hmR0VmxExportHostControlRegs();
9043	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9044
9045	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9046	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9047
9048	rc = hmR0VmxExportHostMsrs(pVCpu);
9049	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9050
9051	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9052	}
9053	return rc;
9054	}
9055
9056
9057	/**
9058	* Saves the host state in the VMCS host-state.
9059	*
9060	* @returns VBox status code.
9061	* @param pVCpu The cross context virtual CPU structure.
9062	*
9063	* @remarks No-long-jump zone!!!
9064	*/
9065	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9066	{
9067	AssertPtr(pVCpu);
9068	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9069
9070	/*
9071	* Export the host state here while entering HM context.
9072	* When thread-context hooks are used, we might get preempted and have to re-save the host
9073	* state but most of the time we won't be, so do it here before we disable interrupts.
9074	*/
9075	return hmR0VmxExportHostState(pVCpu);
9076	}
9077
9078
9079	/**
9080	* Exports the guest state into the VMCS guest-state area.
9081	*
9082	* The will typically be done before VM-entry when the guest-CPU state and the
9083	* VMCS state may potentially be out of sync.
9084	*
9085	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9086	* VM-entry controls.
9087	* Sets up the appropriate VMX non-root function to execute guest code based on
9088	* the guest CPU mode.
9089	*
9090	* @returns VBox strict status code.
9091	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9092	* without unrestricted guest execution and the VMMDev is not presently
9093	* mapped (e.g. EFI32).
9094	*
9095	* @param pVCpu The cross context virtual CPU structure.
9096	* @param pVmxTransient The VMX-transient structure.
9097	*
9098	* @remarks No-long-jump zone!!!
9099	*/
9100	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9101	{
9102	AssertPtr(pVCpu);
9103	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9104	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9105
9106	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9107
9108	/*
9109	* Determine real-on-v86 mode.
9110	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9111	*/
9112	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9113	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9114	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9115	pVmcsInfo->RealMode. fRealOnV86Active = false;
9116	else
9117	{
9118	Assert(!pVmxTransient->fIsNestedGuest);
9119	pVmcsInfo->RealMode.fRealOnV86Active = true;
9120	}
9121
9122	/*
9123	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9124	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9125	*/
9126	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9127	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9128	* be a need to evaluate this everytime since I'm pretty sure we intercept
9129	* all guest paging mode changes. */
9130	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9131	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9132
9133	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9134	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9135
9136	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9137	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9138
9139	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9140	if (rcStrict == VINF_SUCCESS)
9141	{ /* likely */ }
9142	else
9143	{
9144	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9145	return rcStrict;
9146	}
9147
9148	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9149	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9150
9151	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9152	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9153
9154	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9155	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9156
9157	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9158	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9159
9160	rc = hmR0VmxExportGuestRip(pVCpu);
9161	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9162	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9163	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9164
9165	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9166	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9167	\| HM_CHANGED_GUEST_CR2
9168	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9169	\| HM_CHANGED_GUEST_X87
9170	\| HM_CHANGED_GUEST_SSE_AVX
9171	\| HM_CHANGED_GUEST_OTHER_XSAVE
9172	\| HM_CHANGED_GUEST_XCRx
9173	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9174	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9175	\| HM_CHANGED_GUEST_TSC_AUX
9176	\| HM_CHANGED_GUEST_OTHER_MSRS
9177	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9178	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9179
9180	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9181	return rc;
9182	}
9183
9184
9185	/**
9186	* Exports the state shared between the host and guest into the VMCS.
9187	*
9188	* @param pVCpu The cross context virtual CPU structure.
9189	* @param pVmxTransient The VMX-transient structure.
9190	*
9191	* @remarks No-long-jump zone!!!
9192	*/
9193	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9194	{
9195	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9196	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9197
9198	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9199	{
9200	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9201	AssertRC(rc);
9202	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9203
9204	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9205	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9206	{
9207	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9208	AssertRC(rc);
9209	}
9210	}
9211
9212	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9213	{
9214	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9215	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9216	}
9217
9218	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9219	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9220	}
9221
9222
9223	/**
9224	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9225	*
9226	* @returns Strict VBox status code (i.e. informational status codes too).
9227	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9228	* without unrestricted guest execution and the VMMDev is not presently
9229	* mapped (e.g. EFI32).
9230	*
9231	* @param pVCpu The cross context virtual CPU structure.
9232	* @param pVmxTransient The VMX-transient structure.
9233	*
9234	* @remarks No-long-jump zone!!!
9235	*/
9236	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9237	{
9238	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9239	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9240	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9241
9242	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9243	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9244	#endif
9245
9246	/*
9247	* For many exits it's only RIP that changes and hence try to export it first
9248	* without going through a lot of change flag checks.
9249	*/
9250	VBOXSTRICTRC rcStrict;
9251	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9252	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9253	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9254	{
9255	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9256	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9257	{ /* likely */}
9258	else
9259	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9261	}
9262	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9263	{
9264	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9265	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9266	{ /* likely */}
9267	else
9268	{
9269	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9270	VBOXSTRICTRC_VAL(rcStrict)));
9271	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9272	return rcStrict;
9273	}
9274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9275	}
9276	else
9277	rcStrict = VINF_SUCCESS;
9278
9279	#ifdef VBOX_STRICT
9280	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9281	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9282	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9283	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9284	("fCtxChanged=%#RX64\n", fCtxChanged));
9285	#endif
9286	return rcStrict;
9287	}
9288
9289
9290	/**
9291	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9292	* and update error record fields accordingly.
9293	*
9294	* @return VMX_IGS_* return codes.
9295	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9296	* wrong with the guest state.
9297	*
9298	* @param pVCpu The cross context virtual CPU structure.
9299	* @param pVmcsInfo The VMCS info. object.
9300	*
9301	* @remarks This function assumes our cache of the VMCS controls
9302	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9303	*/
9304	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9305	{
9306	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9307	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9308	uError = (err); \
9309	break; \
9310	} else do { } while (0)
9311
9312	int rc;
9313	PVM pVM = pVCpu->CTX_SUFF(pVM);
9314	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9315	uint32_t uError = VMX_IGS_ERROR;
9316	uint32_t u32Val;
9317	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9318
9319	do
9320	{
9321	/*
9322	* CR0.
9323	*/
9324	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9325	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9326	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9327	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9328	if (fUnrestrictedGuest)
9329	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9330
9331	uint32_t u32GuestCr0;
9332	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9333	AssertRCBreak(rc);
9334	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9335	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9336	if ( !fUnrestrictedGuest
9337	&& (u32GuestCr0 & X86_CR0_PG)
9338	&& !(u32GuestCr0 & X86_CR0_PE))
9339	{
9340	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9341	}
9342
9343	/*
9344	* CR4.
9345	*/
9346	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9347	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9348
9349	uint32_t u32GuestCr4;
9350	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9351	AssertRCBreak(rc);
9352	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9353	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9354
9355	/*
9356	* IA32_DEBUGCTL MSR.
9357	*/
9358	uint64_t u64Val;
9359	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9360	AssertRCBreak(rc);
9361	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9362	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9363	{
9364	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9365	}
9366	uint64_t u64DebugCtlMsr = u64Val;
9367
9368	#ifdef VBOX_STRICT
9369	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9370	AssertRCBreak(rc);
9371	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9372	#endif
9373	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9374
9375	/*
9376	* RIP and RFLAGS.
9377	*/
9378	uint32_t u32Eflags;
9379	#if HC_ARCH_BITS == 64
9380	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9381	AssertRCBreak(rc);
9382	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9383	if ( !fLongModeGuest
9384	\|\| !pCtx->cs.Attr.n.u1Long)
9385	{
9386	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9387	}
9388	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9389	* must be identical if the "IA-32e mode guest" VM-entry
9390	* control is 1 and CS.L is 1. No check applies if the
9391	* CPU supports 64 linear-address bits. */
9392
9393	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9394	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9395	AssertRCBreak(rc);
9396	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9397	VMX_IGS_RFLAGS_RESERVED);
9398	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9399	u32Eflags = u64Val;
9400	#else
9401	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9402	AssertRCBreak(rc);
9403	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9404	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9405	#endif
9406
9407	if ( fLongModeGuest
9408	\|\| ( fUnrestrictedGuest
9409	&& !(u32GuestCr0 & X86_CR0_PE)))
9410	{
9411	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9412	}
9413
9414	uint32_t u32EntryInfo;
9415	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9416	AssertRCBreak(rc);
9417	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9418	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9419	{
9420	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9421	}
9422
9423	/*
9424	* 64-bit checks.
9425	*/
9426	#if HC_ARCH_BITS == 64
9427	if (fLongModeGuest)
9428	{
9429	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9430	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9431	}
9432
9433	if ( !fLongModeGuest
9434	&& (u32GuestCr4 & X86_CR4_PCIDE))
9435	{
9436	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9437	}
9438
9439	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9440	* 51:32 beyond the processor's physical-address width are 0. */
9441
9442	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9443	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9444	{
9445	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9446	}
9447
9448	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9449	AssertRCBreak(rc);
9450	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9451
9452	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9453	AssertRCBreak(rc);
9454	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9455	#endif
9456
9457	/*
9458	* PERF_GLOBAL MSR.
9459	*/
9460	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9461	{
9462	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9463	AssertRCBreak(rc);
9464	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9465	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9466	}
9467
9468	/*
9469	* PAT MSR.
9470	*/
9471	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9472	{
9473	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9474	AssertRCBreak(rc);
9475	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9476	for (unsigned i = 0; i < 8; i++)
9477	{
9478	uint8_t u8Val = (u64Val & 0xff);
9479	if ( u8Val != 0 /* UC */
9480	&& u8Val != 1 /* WC */
9481	&& u8Val != 4 /* WT */
9482	&& u8Val != 5 /* WP */
9483	&& u8Val != 6 /* WB */
9484	&& u8Val != 7 /* UC- */)
9485	{
9486	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9487	}
9488	u64Val >>= 8;
9489	}
9490	}
9491
9492	/*
9493	* EFER MSR.
9494	*/
9495	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9496	{
9497	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9498	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9499	AssertRCBreak(rc);
9500	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9501	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9502	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9503	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9504	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9505	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9506	* iemVmxVmentryCheckGuestState(). */
9507	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9508	\|\| !(u32GuestCr0 & X86_CR0_PG)
9509	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9510	VMX_IGS_EFER_LMA_LME_MISMATCH);
9511	}
9512
9513	/*
9514	* Segment registers.
9515	*/
9516	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9517	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9518	if (!(u32Eflags & X86_EFL_VM))
9519	{
9520	/* CS */
9521	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9522	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9523	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9524	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9525	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9526	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9527	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9528	/* CS cannot be loaded with NULL in protected mode. */
9529	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9530	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9531	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9532	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9533	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9534	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9535	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9536	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9537	else
9538	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9539
9540	/* SS */
9541	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9542	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9543	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9544	if ( !(pCtx->cr0 & X86_CR0_PE)
9545	\|\| pCtx->cs.Attr.n.u4Type == 3)
9546	{
9547	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9548	}
9549	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9550	{
9551	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9552	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9553	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9554	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9555	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9556	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9557	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9558	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9559	}
9560
9561	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9562	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9563	{
9564	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9565	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9566	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9567	\|\| pCtx->ds.Attr.n.u4Type > 11
9568	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9569	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9570	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9571	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9572	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9573	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9574	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9575	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9576	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9577	}
9578	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9579	{
9580	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9581	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9582	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9583	\|\| pCtx->es.Attr.n.u4Type > 11
9584	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9585	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9586	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9587	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9588	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9589	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9590	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9591	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9592	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9593	}
9594	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9595	{
9596	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9597	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9598	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9599	\|\| pCtx->fs.Attr.n.u4Type > 11
9600	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9601	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9602	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9603	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9604	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9605	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9606	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9607	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9608	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9609	}
9610	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9611	{
9612	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9613	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9614	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9615	\|\| pCtx->gs.Attr.n.u4Type > 11
9616	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9617	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9618	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9619	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9620	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9621	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9622	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9623	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9624	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9625	}
9626	/* 64-bit capable CPUs. */
9627	#if HC_ARCH_BITS == 64
9628	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9629	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9630	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9631	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9632	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9633	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9634	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9635	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9636	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9637	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9638	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9639	#endif
9640	}
9641	else
9642	{
9643	/* V86 mode checks. */
9644	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9645	if (pVmcsInfo->RealMode.fRealOnV86Active)
9646	{
9647	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9648	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9649	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9650	}
9651	else
9652	{
9653	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9654	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9655	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9656	}
9657
9658	/* CS */
9659	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9660	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9661	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9662	/* SS */
9663	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9664	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9665	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9666	/* DS */
9667	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9668	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9669	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9670	/* ES */
9671	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9672	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9673	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9674	/* FS */
9675	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9676	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9677	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9678	/* GS */
9679	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9680	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9681	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9682	/* 64-bit capable CPUs. */
9683	#if HC_ARCH_BITS == 64
9684	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9685	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9686	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9687	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9688	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9689	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9690	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9691	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9692	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9693	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9694	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9695	#endif
9696	}
9697
9698	/*
9699	* TR.
9700	*/
9701	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9702	/* 64-bit capable CPUs. */
9703	#if HC_ARCH_BITS == 64
9704	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9705	#endif
9706	if (fLongModeGuest)
9707	{
9708	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9709	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9710	}
9711	else
9712	{
9713	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9714	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9715	VMX_IGS_TR_ATTR_TYPE_INVALID);
9716	}
9717	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9718	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9719	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9720	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9721	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9722	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9723	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9724	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9725
9726	/*
9727	* GDTR and IDTR.
9728	*/
9729	#if HC_ARCH_BITS == 64
9730	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9731	AssertRCBreak(rc);
9732	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9733
9734	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9735	AssertRCBreak(rc);
9736	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9737	#endif
9738
9739	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9740	AssertRCBreak(rc);
9741	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9742
9743	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9744	AssertRCBreak(rc);
9745	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9746
9747	/*
9748	* Guest Non-Register State.
9749	*/
9750	/* Activity State. */
9751	uint32_t u32ActivityState;
9752	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9753	AssertRCBreak(rc);
9754	HMVMX_CHECK_BREAK( !u32ActivityState
9755	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9756	VMX_IGS_ACTIVITY_STATE_INVALID);
9757	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9758	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9759	uint32_t u32IntrState;
9760	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9761	AssertRCBreak(rc);
9762	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9763	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9764	{
9765	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9766	}
9767
9768	/** @todo Activity state and injecting interrupts. Left as a todo since we
9769	* currently don't use activity states but ACTIVE. */
9770
9771	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9772	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9773
9774	/* Guest interruptibility-state. */
9775	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9776	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9777	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9778	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9779	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9780	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9781	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9782	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9783	{
9784	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9785	{
9786	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9787	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9788	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9789	}
9790	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9791	{
9792	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9793	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9794	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9795	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9796	}
9797	}
9798	/** @todo Assumes the processor is not in SMM. */
9799	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9800	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9801	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9802	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9803	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9804	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9805	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9806	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9807	{
9808	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9809	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9810	}
9811
9812	/* Pending debug exceptions. */
9813	#if HC_ARCH_BITS == 64
9814	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9815	AssertRCBreak(rc);
9816	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9817	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9818	u32Val = u64Val; /* For pending debug exceptions checks below. */
9819	#else
9820	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9821	AssertRCBreak(rc);
9822	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9823	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9824	#endif
9825
9826	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9827	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9828	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9829	{
9830	if ( (u32Eflags & X86_EFL_TF)
9831	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9832	{
9833	/* Bit 14 is PendingDebug.BS. */
9834	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9835	}
9836	if ( !(u32Eflags & X86_EFL_TF)
9837	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9838	{
9839	/* Bit 14 is PendingDebug.BS. */
9840	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9841	}
9842	}
9843
9844	/* VMCS link pointer. */
9845	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9846	AssertRCBreak(rc);
9847	if (u64Val != UINT64_C(0xffffffffffffffff))
9848	{
9849	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9850	/** @todo Bits beyond the processor's physical-address width MBZ. */
9851	/** @todo 32-bit located in memory referenced by value of this field (as a
9852	* physical address) must contain the processor's VMCS revision ID. */
9853	/** @todo SMM checks. */
9854	}
9855
9856	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9857	* not using nested paging? */
9858	if ( pVM->hm.s.fNestedPaging
9859	&& !fLongModeGuest
9860	&& CPUMIsGuestInPAEModeEx(pCtx))
9861	{
9862	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9863	AssertRCBreak(rc);
9864	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9865
9866	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9867	AssertRCBreak(rc);
9868	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9869
9870	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9871	AssertRCBreak(rc);
9872	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9873
9874	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9875	AssertRCBreak(rc);
9876	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9877	}
9878
9879	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9880	if (uError == VMX_IGS_ERROR)
9881	uError = VMX_IGS_REASON_NOT_FOUND;
9882	} while (0);
9883
9884	pVCpu->hm.s.u32HMError = uError;
9885	return uError;
9886
9887	#undef HMVMX_ERROR_BREAK
9888	#undef HMVMX_CHECK_BREAK
9889	}
9890
9891
9892	/**
9893	* Setup the APIC-access page for virtualizing APIC access.
9894	*
9895	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9896	* this not done as part of exporting guest state, see @bugref{8721}.
9897	*
9898	* @returns VBox status code.
9899	* @param pVCpu The cross context virtual CPU structure.
9900	*/
9901	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9902	{
9903	PVM pVM = pVCpu->CTX_SUFF(pVM);
9904	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9905
9906	Assert(PDMHasApic(pVM));
9907	Assert(u64MsrApicBase);
9908
9909	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9910	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9911
9912	/* Unalias any existing mapping. */
9913	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9914	AssertRCReturn(rc, rc);
9915
9916	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9917	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9918	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9919	AssertRCReturn(rc, rc);
9920
9921	/* Update the per-VCPU cache of the APIC base MSR. */
9922	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9923	return VINF_SUCCESS;
9924	}
9925
9926
9927	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9928	/**
9929	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9930	* nested-guest using hardware-assisted VMX.
9931	*
9932	* @param pVCpu The cross context virtual CPU structure.
9933	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9934	* @param pVmcsInfoGst The guest VMCS info. object.
9935	*/
9936	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9937	{
9938	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9939	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9940	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9941	Assert(pu64MsrBitmapNstGst);
9942	Assert(pu64MsrBitmapGst);
9943	Assert(pu64MsrBitmap);
9944
9945	/*
9946	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9947	* MSR that is intercepted by the guest is also intercepted while executing the
9948	* nested-guest using hardware-assisted VMX.
9949	*/
9950	uint32_t const cbFrag = sizeof(uint64_t);
9951	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9952	for (uint32_t i = 0; i <= cFrags; i++)
9953	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9954	}
9955
9956
9957	/**
9958	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9959	* hardware-assisted VMX execution of the nested-guest.
9960	*
9961	* For a guest, we don't modify these controls once we set up the VMCS and hence
9962	* this function is never called.
9963	*
9964	* For nested-guests since the guest hypervisor provides these controls on every
9965	* nested-guest VM-entry and could potentially change them everytime we need to
9966	* merge them before every nested-guest VM-entry.
9967	*
9968	* @returns VBox status code.
9969	* @param pVCpu The cross context virtual CPU structure.
9970	*/
9971	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9972	{
9973	PVM pVM = pVCpu->CTX_SUFF(pVM);
9974	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9975	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9976	Assert(pVmcsNstGst);
9977
9978	/*
9979	* Merge the controls with the requirements of the guest VMCS.
9980	*
9981	* We do not need to validate the nested-guest VMX features specified in the
9982	* nested-guest VMCS with the features supported by the physical CPU as it's
9983	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9984	*
9985	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9986	* guest are derived from the VMX features supported by the physical CPU.
9987	*/
9988
9989	/* Pin-based VM-execution controls. */
9990	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9991
9992	/* Processor-based VM-execution controls. */
9993	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9994	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9995	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9996	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9997	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9998
9999	/* Secondary processor-based VM-execution controls. */
10000	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10001	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10002	\| VMX_PROC_CTLS2_INVPCID
10003	\| VMX_PROC_CTLS2_RDTSCP
10004	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10005	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10006	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10007	\| VMX_PROC_CTLS2_VMFUNC));
10008
10009	/*
10010	* VM-entry controls:
10011	* These controls contains state that depends on the nested-guest state (primarily
10012	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10013	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10014	* properly continue executing the nested-guest if the EFER MSR changes but does not
10015	* cause a nested-guest VM-exits.
10016	*
10017	* VM-exit controls:
10018	* These controls specify the host state on return. We cannot use the controls from
10019	* the nested-hypervisor state as is as it would contain the guest state rather than
10020	* the host state. Since the host state is subject to change (e.g. preemption, trips
10021	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10022	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10023	*
10024	* VM-entry MSR-load:
10025	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10026	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10027	*
10028	* VM-exit MSR-store:
10029	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10030	* context back into the VM-exit MSR-store area.
10031	*
10032	* VM-exit MSR-load areas:
10033	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10034	* we can entirely ignore what the nested-hypervisor wants to load here.
10035	*/
10036
10037	/*
10038	* Exception bitmap.
10039	*
10040	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10041	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10042	* keep the code more flexible if intercepting exceptions become more dynamic in
10043	* the future we do it as part of exporting the nested-guest state.
10044	*/
10045	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10046
10047	/*
10048	* CR0/CR4 guest/host mask.
10049	*
10050	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10051	* must cause VM-exits, so we need to merge them here.
10052	*/
10053	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10054	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10055
10056	/*
10057	* Page-fault error-code mask and match.
10058	*
10059	* Although we require unrestricted guest execution (and thereby nested-paging) for
10060	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10061	* normally intercept #PFs, it might intercept them for debugging purposes.
10062	*
10063	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10064	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10065	*/
10066	uint32_t u32XcptPFMask;
10067	uint32_t u32XcptPFMatch;
10068	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10069	{
10070	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10071	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10072	}
10073	else
10074	{
10075	u32XcptPFMask = 0;
10076	u32XcptPFMatch = 0;
10077	}
10078
10079	/*
10080	* Pause-Loop exiting.
10081	*/
10082	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10083	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10084
10085	/*
10086	* I/O Bitmap.
10087	*
10088	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10089	* always intercept all I/O port accesses.
10090	*/
10091	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10092
10093	/*
10094	* APIC-access page.
10095	*
10096	* The APIC-access page address has already been initialized while setting up the
10097	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10098	* should not be on any consequence to the host or to the guest for that matter, but
10099	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10100	* emulation to keep it simple.
10101	*/
10102
10103	/*
10104	* Virtual-APIC page and TPR threshold.
10105	*
10106	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10107	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10108	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10109	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10110	*/
10111	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10112	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10113	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10114	{
10115	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10116
10117	/*
10118	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10119	* we would fail to obtain a valid host-physical address for its guest-physical
10120	* address.
10121	*
10122	* We currently do not support this scenario. Maybe in the future if there is a
10123	* pressing need we can explore making this particular set of conditions work.
10124	* Right now we just cause a VM-entry failure.
10125	*
10126	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10127	* so should not really failure at the moment.
10128	*/
10129	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10130	}
10131	else
10132	{
10133	/*
10134	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10135	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10136	* be taken care of by EPT/shadow paging.
10137	*/
10138	if (pVM->hm.s.fAllow64BitGuests)
10139	{
10140	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10141	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10142	}
10143	}
10144
10145	/*
10146	* Validate basic assumptions.
10147	*/
10148	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10149	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10150	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10151	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10152
10153	/*
10154	* Commit it to the nested-guest VMCS.
10155	*/
10156	int rc = VINF_SUCCESS;
10157	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10158	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10159	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10160	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10161	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10162	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10163	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10164	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10165	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10166	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10167	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10168	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10169	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10170	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10171	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10172	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10173	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10174	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10175	{
10176	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10177	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10178	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10179	}
10180	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10181	{
10182	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10183	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10184	}
10185	AssertRCReturn(rc, rc);
10186
10187	/*
10188	* Update the nested-guest VMCS cache.
10189	*/
10190	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10191	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10192	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10193	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10194	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10195	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10196	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10197	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10198	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10199
10200	/*
10201	* MSR bitmap.
10202	*
10203	* The MSR bitmap address has already been initialized while setting up the
10204	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10205	*/
10206	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10207	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10208
10209	return VINF_SUCCESS;
10210	}
10211	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10212
10213
10214	/**
10215	* Does the preparations before executing guest code in VT-x.
10216	*
10217	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10218	* recompiler/IEM. We must be cautious what we do here regarding committing
10219	* guest-state information into the VMCS assuming we assuredly execute the
10220	* guest in VT-x mode.
10221	*
10222	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10223	* the common-state (TRPM/forceflags), we must undo those changes so that the
10224	* recompiler/IEM can (and should) use them when it resumes guest execution.
10225	* Otherwise such operations must be done when we can no longer exit to ring-3.
10226	*
10227	* @returns Strict VBox status code (i.e. informational status codes too).
10228	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10229	* have been disabled.
10230	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10231	* double-fault into the guest.
10232	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10233	* dispatched directly.
10234	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10235	*
10236	* @param pVCpu The cross context virtual CPU structure.
10237	* @param pVmxTransient The VMX-transient structure.
10238	* @param fStepping Whether we are single-stepping the guest in the
10239	* hypervisor debugger. Makes us ignore some of the reasons
10240	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10241	* if event dispatching took place.
10242	*/
10243	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10244	{
10245	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10246
10247	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10248	if (pVmxTransient->fIsNestedGuest)
10249	{
10250	RT_NOREF2(pVCpu, fStepping);
10251	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10252	return VINF_EM_RESCHEDULE_REM;
10253	}
10254	#endif
10255
10256	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10257	PGMRZDynMapFlushAutoSet(pVCpu);
10258	#endif
10259
10260	/*
10261	* Check and process force flag actions, some of which might require us to go back to ring-3.
10262	*/
10263	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10264	if (rcStrict == VINF_SUCCESS)
10265	{ /* FFs don't get set all the time. */ }
10266	else
10267	return rcStrict;
10268
10269	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10270	/*
10271	* Switch to the nested-guest VMCS as we may have transitioned into executing
10272	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10273	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10274	*
10275	* We do this as late as possible to minimize (though not completely remove)
10276	* clearing/loading VMCS again due to premature trips to ring-3 above.
10277	*/
10278	if (pVmxTransient->fIsNestedGuest)
10279	{
10280	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10281	{
10282	/*
10283	* Ensure we have synced everything from the guest VMCS and also flag that
10284	* that we need to export the full (nested) guest-CPU context to the
10285	* nested-guest VMCS.
10286	*/
10287	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10288	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10289
10290	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10291	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10292	if (RT_LIKELY(rc == VINF_SUCCESS))
10293	{
10294	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10295	ASMSetFlags(fEFlags);
10296	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10297
10298	/*
10299	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10300	* flag that we need to update the host MSR values there. Even if we decide
10301	* in the future to share the VM-exit MSR-store area page with the guest,
10302	* if its content differs, we would have to update the host MSRs anyway.
10303	*/
10304	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10305	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10306	}
10307	else
10308	{
10309	ASMSetFlags(fEFlags);
10310	return rc;
10311	}
10312	}
10313
10314	/*
10315	* Merge guest VMCS controls with the nested-guest VMCS controls.
10316	*
10317	* Even if we have not executed the guest prior to this (e.g. when resuming
10318	* from a saved state), we should be okay with merging controls as we
10319	* initialize the guest VMCS controls as part of VM setup phase.
10320	*/
10321	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10322	{
10323	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10324	AssertRCReturn(rc, rc);
10325	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10326	}
10327	}
10328	#endif
10329
10330	/*
10331	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10332	* We look at the guest VMCS control here as we always set it when supported by
10333	* the physical CPU. Looking at the nested-guest control here would not be
10334	* possible because they are not merged yet.
10335	*/
10336	PVM pVM = pVCpu->CTX_SUFF(pVM);
10337	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10338	Assert(pVmcsInfo);
10339	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10340	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10341	&& PDMHasApic(pVM))
10342	{
10343	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10344	AssertRCReturn(rc, rc);
10345	}
10346
10347	/*
10348	* Evaluate events to be injected into the guest.
10349	*
10350	* Events in TRPM can be injected without inspecting the guest state.
10351	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10352	* guest to cause a VM-exit the next time they are ready to receive the event.
10353	*/
10354	if (TRPMHasTrap(pVCpu))
10355	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10356
10357	uint32_t fIntrState;
10358	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10359
10360	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10361	/*
10362	* While evaluating pending events if something failed (unlikely) or if we were
10363	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10364	*/
10365	if ( rcStrict != VINF_SUCCESS
10366	\|\| ( pVmxTransient->fIsNestedGuest
10367	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10368	return rcStrict;
10369	#endif
10370
10371	/*
10372	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10373	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10374	* also result in triple-faulting the VM.
10375	*
10376	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10377	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10378	*/
10379	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10380	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10381	{ /* likely */ }
10382	else
10383	{
10384	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10385	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10386	return rcStrict;
10387	}
10388
10389	/*
10390	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10391	* import CR3 themselves. We will need to update them here, as even as late as the above
10392	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10393	* the below force flags to be set.
10394	*/
10395	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10396	{
10397	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10398	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10399	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10400	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10401	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10402	}
10403	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10404	{
10405	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10406	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10407	}
10408
10409	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10410	/* Paranoia. */
10411	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10412	#endif
10413
10414	/*
10415	* No longjmps to ring-3 from this point on!!!
10416	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10417	* This also disables flushing of the R0-logger instance (if any).
10418	*/
10419	VMMRZCallRing3Disable(pVCpu);
10420
10421	/*
10422	* Export the guest state bits.
10423	*
10424	* We cannot perform longjmps while loading the guest state because we do not preserve the
10425	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10426	* CPU migration.
10427	*
10428	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10429	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10430	*/
10431	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10432	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10433	{ /* likely */ }
10434	else
10435	{
10436	VMMRZCallRing3Enable(pVCpu);
10437	return rcStrict;
10438	}
10439
10440	/*
10441	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10442	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10443	* preemption disabled for a while. Since this is purely to aid the
10444	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10445	* disable interrupt on NT.
10446	*
10447	* We need to check for force-flags that could've possible been altered since we last
10448	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10449	* see @bugref{6398}).
10450	*
10451	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10452	* to ring-3 before executing guest code.
10453	*/
10454	pVmxTransient->fEFlags = ASMIntDisableFlags();
10455
10456	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10457	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10458	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10459	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10460	{
10461	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10462	{
10463	pVCpu->hm.s.Event.fPending = false;
10464
10465	/*
10466	* We've injected any pending events. This is really the point of no return (to ring-3).
10467	*
10468	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10469	* returns from this function, so don't enable them here.
10470	*/
10471	return VINF_SUCCESS;
10472	}
10473
10474	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10475	rcStrict = VINF_EM_RAW_INTERRUPT;
10476	}
10477	else
10478	{
10479	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10480	rcStrict = VINF_EM_RAW_TO_R3;
10481	}
10482
10483	ASMSetFlags(pVmxTransient->fEFlags);
10484	VMMRZCallRing3Enable(pVCpu);
10485
10486	return rcStrict;
10487	}
10488
10489
10490	/**
10491	* Final preparations before executing guest code using hardware-assisted VMX.
10492	*
10493	* We can no longer get preempted to a different host CPU and there are no returns
10494	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10495	* failures), this function is not intended to fail sans unrecoverable hardware
10496	* errors.
10497	*
10498	* @param pVCpu The cross context virtual CPU structure.
10499	* @param pVmxTransient The VMX-transient structure.
10500	*
10501	* @remarks Called with preemption disabled.
10502	* @remarks No-long-jump zone!!!
10503	*/
10504	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10505	{
10506	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10507	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10508	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10509	Assert(!pVCpu->hm.s.Event.fPending);
10510
10511	/*
10512	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10513	*/
10514	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10515	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10516
10517	PVM pVM = pVCpu->CTX_SUFF(pVM);
10518	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10519
10520	if (!CPUMIsGuestFPUStateActive(pVCpu))
10521	{
10522	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10523	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10524	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10525	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10526	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10527	}
10528
10529	/*
10530	* Re-save the host state bits as we may've been preempted (only happens when
10531	* thread-context hooks are used or when the VM start function changes).
10532	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10533	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10534	* see @bugref{8432}.
10535	*
10536	* This may also happen when switching to/from a nested-guest VMCS without leaving
10537	* ring-0.
10538	*/
10539	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10540	{
10541	int rc = hmR0VmxExportHostState(pVCpu);
10542	AssertRC(rc);
10543	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10544	}
10545	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10546
10547	/*
10548	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10549	*/
10550	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10551	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10552	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10553
10554	/*
10555	* Store status of the shared guest/host debug state at the time of VM-entry.
10556	*/
10557	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10558	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10559	{
10560	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10561	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10562	}
10563	else
10564	#endif
10565	{
10566	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10567	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10568	}
10569
10570	/*
10571	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10572	* more than one conditional check. The post-run side of our code shall determine
10573	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10574	*/
10575	if (pVmcsInfo->pbVirtApic)
10576	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10577
10578	/*
10579	* Update the host MSRs values in the VM-exit MSR-load area.
10580	*/
10581	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10582	{
10583	if (pVmcsInfo->cExitMsrLoad > 0)
10584	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10585	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10586	}
10587
10588	/*
10589	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10590	* VMX-preemption timer based on the next virtual sync clock deadline.
10591	*/
10592	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10593	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10594	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10595	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10596	{
10597	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10598	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10599	}
10600
10601	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10602	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10603	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10604	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10605
10606	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10607
10608	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10609	as we're about to start executing the guest . */
10610
10611	/*
10612	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10613	*
10614	* This is done this late as updating the TSC offsetting/preemption timer above
10615	* figures out if we can skip intercepting RDTSCP by calculating the number of
10616	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10617	*/
10618	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10619	{
10620	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10621	{
10622	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10623	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10624	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10625	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10626	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10627	AssertRC(rc);
10628	}
10629	else
10630	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10631	}
10632
10633	#ifdef VBOX_STRICT
10634	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10635	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10636	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10637	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10638	#endif
10639
10640	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10641	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10642	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10643	* see @bugref{9180#c54}. */
10644	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10645	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10646	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10647	#endif
10648	}
10649
10650
10651	/**
10652	* First C routine invoked after running guest code using hardware-assisted VMX.
10653	*
10654	* @param pVCpu The cross context virtual CPU structure.
10655	* @param pVmxTransient The VMX-transient structure.
10656	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10657	*
10658	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10659	*
10660	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10661	* unconditionally when it is safe to do so.
10662	*/
10663	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10664	{
10665	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10666
10667	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10668	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10669	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10670	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10671	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10672	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10673
10674	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10675	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10676	{
10677	uint64_t uGstTsc;
10678	if (!pVmxTransient->fIsNestedGuest)
10679	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10680	else
10681	{
10682	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10683	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10684	}
10685	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10686	}
10687
10688	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10689	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10690	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10691
10692	#if HC_ARCH_BITS == 64
10693	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10694	#endif
10695	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10696	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10697	and we need to leave it alone here. */
10698	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10699	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10700	#else
10701	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10702	#endif
10703	#ifdef VBOX_STRICT
10704	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10705	#endif
10706	Assert(!ASMIntAreEnabled());
10707	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10708	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10709
10710	/*
10711	* Save the basic VM-exit reason and check if the VM-entry failed.
10712	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10713	*/
10714	uint32_t uExitReason;
10715	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10716	AssertRC(rc);
10717	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10718	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10719
10720	/*
10721	* Check if VMLAUNCH/VMRESUME succeeded.
10722	* If this failed, we cause a guru meditation and cease further execution.
10723	*
10724	* However, if we are executing a nested-guest we might fail if we use the
10725	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10726	*/
10727	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10728	{
10729	/*
10730	* Update the VM-exit history array here even if the VM-entry failed due to:
10731	* - Invalid guest state.
10732	* - MSR loading.
10733	* - Machine-check event.
10734	*
10735	* In any of the above cases we will still have a "valid" VM-exit reason
10736	* despite @a fVMEntryFailed being false.
10737	*
10738	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10739	*
10740	* Note! We don't have CS or RIP at this point. Will probably address that later
10741	* by amending the history entry added here.
10742	*/
10743	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10744	UINT64_MAX, uHostTsc);
10745
10746	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10747	{
10748	VMMRZCallRing3Enable(pVCpu);
10749
10750	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10751	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10752
10753	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10754	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10755	AssertRC(rc);
10756	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10757	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10758	AssertRC(rc);
10759	#else
10760	/*
10761	* Import the guest-interruptibility state always as we need it while evaluating
10762	* injecting events on re-entry.
10763	*
10764	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10765	* checking for real-mode while exporting the state because all bits that cause
10766	* mode changes wrt CR0 are intercepted.
10767	*/
10768	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10769	AssertRC(rc);
10770	#endif
10771
10772	/*
10773	* Sync the TPR shadow with our APIC state.
10774	*/
10775	if ( !pVmxTransient->fIsNestedGuest
10776	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10777	{
10778	Assert(pVmcsInfo->pbVirtApic);
10779	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10780	{
10781	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10782	AssertRC(rc);
10783	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10784	}
10785	}
10786
10787	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10788	return;
10789	}
10790	}
10791	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10792	else if (pVmxTransient->fIsNestedGuest)
10793	{
10794	# if 0
10795	/*
10796	* Copy the VM-instruction error field to the guest VMCS.
10797	*/
10798	/** @todo NSTVMX: Verify we're using the fast path. */
10799	uint32_t u32RoVmInstrError;
10800	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10801	AssertRCReturn(rc, rc);
10802	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10803	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10804	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10805	# else
10806	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10807	# endif
10808	}
10809	#endif
10810	else
10811	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10812
10813	VMMRZCallRing3Enable(pVCpu);
10814	}
10815
10816
10817	/**
10818	* Runs the guest code using hardware-assisted VMX the normal way.
10819	*
10820	* @returns VBox status code.
10821	* @param pVCpu The cross context virtual CPU structure.
10822	* @param pcLoops Pointer to the number of executed loops.
10823	*/
10824	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10825	{
10826	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10827	Assert(pcLoops);
10828	Assert(*pcLoops <= cMaxResumeLoops);
10829
10830	VMXTRANSIENT VmxTransient;
10831	RT_ZERO(VmxTransient);
10832	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10833
10834	/* Paranoia. */
10835	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10836	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10837
10838	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10839	for (;;)
10840	{
10841	Assert(!HMR0SuspendPending());
10842	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10843	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10844
10845	/*
10846	* Preparatory work for running nested-guest code, this may force us to
10847	* return to ring-3.
10848	*
10849	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10850	*/
10851	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10852	if (rcStrict != VINF_SUCCESS)
10853	break;
10854
10855	/* Interrupts are disabled at this point! */
10856	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10857	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10858	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10859	/* Interrupts are re-enabled at this point! */
10860
10861	/*
10862	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10863	*/
10864	if (RT_SUCCESS(rcRun))
10865	{ /* very likely */ }
10866	else
10867	{
10868	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10869	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10870	return rcRun;
10871	}
10872
10873	/*
10874	* Profile the VM-exit.
10875	*/
10876	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10877	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10878	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10879	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10880	HMVMX_START_EXIT_DISPATCH_PROF();
10881
10882	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10883
10884	/*
10885	* Handle the VM-exit.
10886	*/
10887	#ifdef HMVMX_USE_FUNCTION_TABLE
10888	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10889	#else
10890	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10891	#endif
10892	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10893	if (rcStrict == VINF_SUCCESS)
10894	{
10895	if (++(*pcLoops) <= cMaxResumeLoops)
10896	continue;
10897	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10898	rcStrict = VINF_EM_RAW_INTERRUPT;
10899	}
10900	break;
10901	}
10902
10903	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10904	return rcStrict;
10905	}
10906
10907	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10908	/**
10909	* Runs the nested-guest code using hardware-assisted VMX.
10910	*
10911	* @returns VBox status code.
10912	* @param pVCpu The cross context virtual CPU structure.
10913	* @param pcLoops Pointer to the number of executed loops.
10914	*
10915	* @sa hmR0VmxRunGuestCodeNormal.
10916	*/
10917	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10918	{
10919	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10920	Assert(pcLoops);
10921	Assert(*pcLoops <= cMaxResumeLoops);
10922
10923	VMXTRANSIENT VmxTransient;
10924	RT_ZERO(VmxTransient);
10925	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10926	VmxTransient.fIsNestedGuest = true;
10927
10928	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10929	for (;;)
10930	{
10931	Assert(!HMR0SuspendPending());
10932	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10933	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10934
10935	/*
10936	* Preparatory work for running guest code, this may force us to
10937	* return to ring-3.
10938	*
10939	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10940	*/
10941	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10942	if (rcStrict != VINF_SUCCESS)
10943	break;
10944
10945	/* Interrupts are disabled at this point! */
10946	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10947	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10948	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10949	/* Interrupts are re-enabled at this point! */
10950
10951	/*
10952	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10953	*/
10954	if (RT_SUCCESS(rcRun))
10955	{ /* very likely */ }
10956	else
10957	{
10958	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10959	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10960	return rcRun;
10961	}
10962
10963	/*
10964	* Profile the VM-exit.
10965	*/
10966	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10967	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10968	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10969	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10970	HMVMX_START_EXIT_DISPATCH_PROF();
10971
10972	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10973
10974	/*
10975	* Handle the VM-exit.
10976	*/
10977	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10978	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10979	if ( rcStrict == VINF_SUCCESS
10980	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10981	{
10982	if (++(*pcLoops) <= cMaxResumeLoops)
10983	continue;
10984	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10985	rcStrict = VINF_EM_RAW_INTERRUPT;
10986	}
10987	break;
10988	}
10989
10990	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10991	return rcStrict;
10992	}
10993	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10994
10995
10996	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10997	* probes.
10998	*
10999	* The following few functions and associated structure contains the bloat
11000	* necessary for providing detailed debug events and dtrace probes as well as
11001	* reliable host side single stepping. This works on the principle of
11002	* "subclassing" the normal execution loop and workers. We replace the loop
11003	* method completely and override selected helpers to add necessary adjustments
11004	* to their core operation.
11005	*
11006	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11007	* any performance for debug and analysis features.
11008	*
11009	* @{
11010	*/
11011
11012	/**
11013	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11014	* the debug run loop.
11015	*/
11016	typedef struct VMXRUNDBGSTATE
11017	{
11018	/** The RIP we started executing at. This is for detecting that we stepped. */
11019	uint64_t uRipStart;
11020	/** The CS we started executing with. */
11021	uint16_t uCsStart;
11022
11023	/** Whether we've actually modified the 1st execution control field. */
11024	bool fModifiedProcCtls : 1;
11025	/** Whether we've actually modified the 2nd execution control field. */
11026	bool fModifiedProcCtls2 : 1;
11027	/** Whether we've actually modified the exception bitmap. */
11028	bool fModifiedXcptBitmap : 1;
11029
11030	/** We desire the modified the CR0 mask to be cleared. */
11031	bool fClearCr0Mask : 1;
11032	/** We desire the modified the CR4 mask to be cleared. */
11033	bool fClearCr4Mask : 1;
11034	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11035	uint32_t fCpe1Extra;
11036	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11037	uint32_t fCpe1Unwanted;
11038	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11039	uint32_t fCpe2Extra;
11040	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11041	uint32_t bmXcptExtra;
11042	/** The sequence number of the Dtrace provider settings the state was
11043	* configured against. */
11044	uint32_t uDtraceSettingsSeqNo;
11045	/** VM-exits to check (one bit per VM-exit). */
11046	uint32_t bmExitsToCheck[3];
11047
11048	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11049	uint32_t fProcCtlsInitial;
11050	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11051	uint32_t fProcCtls2Initial;
11052	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11053	uint32_t bmXcptInitial;
11054	} VMXRUNDBGSTATE;
11055	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11056	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11057
11058
11059	/**
11060	* Initializes the VMXRUNDBGSTATE structure.
11061	*
11062	* @param pVCpu The cross context virtual CPU structure of the
11063	* calling EMT.
11064	* @param pVmxTransient The VMX-transient structure.
11065	* @param pDbgState The debug state to initialize.
11066	*/
11067	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11068	{
11069	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11070	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11071
11072	pDbgState->fModifiedProcCtls = false;
11073	pDbgState->fModifiedProcCtls2 = false;
11074	pDbgState->fModifiedXcptBitmap = false;
11075	pDbgState->fClearCr0Mask = false;
11076	pDbgState->fClearCr4Mask = false;
11077	pDbgState->fCpe1Extra = 0;
11078	pDbgState->fCpe1Unwanted = 0;
11079	pDbgState->fCpe2Extra = 0;
11080	pDbgState->bmXcptExtra = 0;
11081	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11082	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11083	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11084	}
11085
11086
11087	/**
11088	* Updates the VMSC fields with changes requested by @a pDbgState.
11089	*
11090	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11091	* immediately before executing guest code, i.e. when interrupts are disabled.
11092	* We don't check status codes here as we cannot easily assert or return in the
11093	* latter case.
11094	*
11095	* @param pVCpu The cross context virtual CPU structure.
11096	* @param pVmxTransient The VMX-transient structure.
11097	* @param pDbgState The debug state.
11098	*/
11099	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11100	{
11101	/*
11102	* Ensure desired flags in VMCS control fields are set.
11103	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11104	*
11105	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11106	* there should be no stale data in pCtx at this point.
11107	*/
11108	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11109	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11110	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11111	{
11112	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11113	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11114	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11115	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11116	pDbgState->fModifiedProcCtls = true;
11117	}
11118
11119	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11120	{
11121	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11122	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11123	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11124	pDbgState->fModifiedProcCtls2 = true;
11125	}
11126
11127	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11128	{
11129	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11130	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11131	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11132	pDbgState->fModifiedXcptBitmap = true;
11133	}
11134
11135	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11136	{
11137	pVmcsInfo->u64Cr0Mask = 0;
11138	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11139	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11140	}
11141
11142	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11143	{
11144	pVmcsInfo->u64Cr4Mask = 0;
11145	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11146	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11147	}
11148
11149	NOREF(pVCpu);
11150	}
11151
11152
11153	/**
11154	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11155	* re-entry next time around.
11156	*
11157	* @returns Strict VBox status code (i.e. informational status codes too).
11158	* @param pVCpu The cross context virtual CPU structure.
11159	* @param pVmxTransient The VMX-transient structure.
11160	* @param pDbgState The debug state.
11161	* @param rcStrict The return code from executing the guest using single
11162	* stepping.
11163	*/
11164	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11165	VBOXSTRICTRC rcStrict)
11166	{
11167	/*
11168	* Restore VM-exit control settings as we may not reenter this function the
11169	* next time around.
11170	*/
11171	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11172
11173	/* We reload the initial value, trigger what we can of recalculations the
11174	next time around. From the looks of things, that's all that's required atm. */
11175	if (pDbgState->fModifiedProcCtls)
11176	{
11177	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11178	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11179	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11180	AssertRCReturn(rc2, rc2);
11181	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11182	}
11183
11184	/* We're currently the only ones messing with this one, so just restore the
11185	cached value and reload the field. */
11186	if ( pDbgState->fModifiedProcCtls2
11187	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11188	{
11189	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11190	AssertRCReturn(rc2, rc2);
11191	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11192	}
11193
11194	/* If we've modified the exception bitmap, we restore it and trigger
11195	reloading and partial recalculation the next time around. */
11196	if (pDbgState->fModifiedXcptBitmap)
11197	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11198
11199	return rcStrict;
11200	}
11201
11202
11203	/**
11204	* Configures VM-exit controls for current DBGF and DTrace settings.
11205	*
11206	* This updates @a pDbgState and the VMCS execution control fields to reflect
11207	* the necessary VM-exits demanded by DBGF and DTrace.
11208	*
11209	* @param pVCpu The cross context virtual CPU structure.
11210	* @param pVmxTransient The VMX-transient structure. May update
11211	* fUpdatedTscOffsettingAndPreemptTimer.
11212	* @param pDbgState The debug state.
11213	*/
11214	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11215	{
11216	/*
11217	* Take down the dtrace serial number so we can spot changes.
11218	*/
11219	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11220	ASMCompilerBarrier();
11221
11222	/*
11223	* We'll rebuild most of the middle block of data members (holding the
11224	* current settings) as we go along here, so start by clearing it all.
11225	*/
11226	pDbgState->bmXcptExtra = 0;
11227	pDbgState->fCpe1Extra = 0;
11228	pDbgState->fCpe1Unwanted = 0;
11229	pDbgState->fCpe2Extra = 0;
11230	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11231	pDbgState->bmExitsToCheck[i] = 0;
11232
11233	/*
11234	* Software interrupts (INT XXh) - no idea how to trigger these...
11235	*/
11236	PVM pVM = pVCpu->CTX_SUFF(pVM);
11237	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11238	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11239	{
11240	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11241	}
11242
11243	/*
11244	* INT3 breakpoints - triggered by #BP exceptions.
11245	*/
11246	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11247	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11248
11249	/*
11250	* Exception bitmap and XCPT events+probes.
11251	*/
11252	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11253	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11254	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11255
11256	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11257	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11258	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11259	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11260	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11261	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11262	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11263	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11264	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11265	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11266	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11267	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11268	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11269	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11270	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11271	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11272	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11273	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11274
11275	if (pDbgState->bmXcptExtra)
11276	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11277
11278	/*
11279	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11280	*
11281	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11282	* So, when adding/changing/removing please don't forget to update it.
11283	*
11284	* Some of the macros are picking up local variables to save horizontal space,
11285	* (being able to see it in a table is the lesser evil here).
11286	*/
11287	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11288	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11289	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11290	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11291	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11292	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11293	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11294	} else do { } while (0)
11295	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11296	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11297	{ \
11298	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11299	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11300	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11301	} else do { } while (0)
11302	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11303	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11304	{ \
11305	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11306	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11307	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11308	} else do { } while (0)
11309	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11310	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11311	{ \
11312	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11313	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11314	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11315	} else do { } while (0)
11316
11317	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11318	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11319	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11320	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11321	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11322
11323	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11324	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11325	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11326	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11327	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11328	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11329	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11330	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11331	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11332	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11333	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11334	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11335	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11336	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11337	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11338	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11339	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11340	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11341	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11342	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11343	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11344	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11345	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11346	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11347	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11349	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11350	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11351	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11352	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11353	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11354	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11355	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11356	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11357	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11358	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11359
11360	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11361	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11362	{
11363	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11364	\| CPUMCTX_EXTRN_APIC_TPR);
11365	AssertRC(rc);
11366
11367	#if 0 /** @todo fix me */
11368	pDbgState->fClearCr0Mask = true;
11369	pDbgState->fClearCr4Mask = true;
11370	#endif
11371	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11372	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11373	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11374	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11375	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11376	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11377	require clearing here and in the loop if we start using it. */
11378	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11379	}
11380	else
11381	{
11382	if (pDbgState->fClearCr0Mask)
11383	{
11384	pDbgState->fClearCr0Mask = false;
11385	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11386	}
11387	if (pDbgState->fClearCr4Mask)
11388	{
11389	pDbgState->fClearCr4Mask = false;
11390	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11391	}
11392	}
11393	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11394	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11395
11396	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11397	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11398	{
11399	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11400	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11401	}
11402	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11403	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11404
11405	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11406	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11407	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11408	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11409	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11410	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11411	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11412	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11413	#if 0 /** @todo too slow, fix handler. */
11414	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11415	#endif
11416	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11417
11418	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11419	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11420	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11421	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11422	{
11423	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11424	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11425	}
11426	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11427	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11428	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11430
11431	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11432	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11433	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11434	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11435	{
11436	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11437	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11438	}
11439	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11440	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11441	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11442	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11443
11444	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11445	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11446	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11447	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11448	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11449	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11450	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11451	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11452	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11453	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11454	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11455	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11456	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11457	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11458	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11459	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11460	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11461	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11462	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11463	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11464	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11465	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11466
11467	#undef IS_EITHER_ENABLED
11468	#undef SET_ONLY_XBM_IF_EITHER_EN
11469	#undef SET_CPE1_XBM_IF_EITHER_EN
11470	#undef SET_CPEU_XBM_IF_EITHER_EN
11471	#undef SET_CPE2_XBM_IF_EITHER_EN
11472
11473	/*
11474	* Sanitize the control stuff.
11475	*/
11476	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11477	if (pDbgState->fCpe2Extra)
11478	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11479	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11480	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11481	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11482	{
11483	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11484	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11485	}
11486
11487	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11488	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11489	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11490	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11491	}
11492
11493
11494	/**
11495	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11496	* appropriate.
11497	*
11498	* The caller has checked the VM-exit against the
11499	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11500	* already, so we don't have to do that either.
11501	*
11502	* @returns Strict VBox status code (i.e. informational status codes too).
11503	* @param pVCpu The cross context virtual CPU structure.
11504	* @param pVmxTransient The VMX-transient structure.
11505	* @param uExitReason The VM-exit reason.
11506	*
11507	* @remarks The name of this function is displayed by dtrace, so keep it short
11508	* and to the point. No longer than 33 chars long, please.
11509	*/
11510	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11511	{
11512	/*
11513	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11514	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11515	*
11516	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11517	* does. Must add/change/remove both places. Same ordering, please.
11518	*
11519	* Added/removed events must also be reflected in the next section
11520	* where we dispatch dtrace events.
11521	*/
11522	bool fDtrace1 = false;
11523	bool fDtrace2 = false;
11524	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11525	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11526	uint32_t uEventArg = 0;
11527	#define SET_EXIT(a_EventSubName) \
11528	do { \
11529	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11530	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11531	} while (0)
11532	#define SET_BOTH(a_EventSubName) \
11533	do { \
11534	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11535	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11536	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11537	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11538	} while (0)
11539	switch (uExitReason)
11540	{
11541	case VMX_EXIT_MTF:
11542	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11543
11544	case VMX_EXIT_XCPT_OR_NMI:
11545	{
11546	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11547	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11548	{
11549	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11550	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11551	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11552	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11553	{
11554	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11555	{
11556	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11557	uEventArg = pVmxTransient->uExitIntErrorCode;
11558	}
11559	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11560	switch (enmEvent1)
11561	{
11562	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11563	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11564	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11565	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11566	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11567	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11568	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11569	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11570	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11571	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11572	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11573	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11574	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11575	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11576	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11577	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11578	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11579	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11580	default: break;
11581	}
11582	}
11583	else
11584	AssertFailed();
11585	break;
11586
11587	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11588	uEventArg = idxVector;
11589	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11590	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11591	break;
11592	}
11593	break;
11594	}
11595
11596	case VMX_EXIT_TRIPLE_FAULT:
11597	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11598	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11599	break;
11600	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11601	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11602	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11603	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11604	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11605
11606	/* Instruction specific VM-exits: */
11607	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11608	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11609	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11610	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11611	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11612	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11613	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11614	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11615	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11616	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11617	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11618	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11619	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11620	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11621	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11622	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11623	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11624	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11625	case VMX_EXIT_MOV_CRX:
11626	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11627	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11628	SET_BOTH(CRX_READ);
11629	else
11630	SET_BOTH(CRX_WRITE);
11631	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11632	break;
11633	case VMX_EXIT_MOV_DRX:
11634	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11635	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11636	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11637	SET_BOTH(DRX_READ);
11638	else
11639	SET_BOTH(DRX_WRITE);
11640	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11641	break;
11642	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11643	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11644	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11645	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11646	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11647	case VMX_EXIT_GDTR_IDTR_ACCESS:
11648	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11649	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11650	{
11651	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11652	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11653	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11654	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11655	}
11656	break;
11657
11658	case VMX_EXIT_LDTR_TR_ACCESS:
11659	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11660	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11661	{
11662	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11663	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11664	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11665	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11666	}
11667	break;
11668
11669	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11670	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11671	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11672	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11673	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11674	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11675	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11676	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11677	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11678	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11679	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11680
11681	/* Events that aren't relevant at this point. */
11682	case VMX_EXIT_EXT_INT:
11683	case VMX_EXIT_INT_WINDOW:
11684	case VMX_EXIT_NMI_WINDOW:
11685	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11686	case VMX_EXIT_PREEMPT_TIMER:
11687	case VMX_EXIT_IO_INSTR:
11688	break;
11689
11690	/* Errors and unexpected events. */
11691	case VMX_EXIT_INIT_SIGNAL:
11692	case VMX_EXIT_SIPI:
11693	case VMX_EXIT_IO_SMI:
11694	case VMX_EXIT_SMI:
11695	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11696	case VMX_EXIT_ERR_MSR_LOAD:
11697	case VMX_EXIT_ERR_MACHINE_CHECK:
11698	case VMX_EXIT_PML_FULL:
11699	case VMX_EXIT_VIRTUALIZED_EOI:
11700	break;
11701
11702	default:
11703	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11704	break;
11705	}
11706	#undef SET_BOTH
11707	#undef SET_EXIT
11708
11709	/*
11710	* Dtrace tracepoints go first. We do them here at once so we don't
11711	* have to copy the guest state saving and stuff a few dozen times.
11712	* Down side is that we've got to repeat the switch, though this time
11713	* we use enmEvent since the probes are a subset of what DBGF does.
11714	*/
11715	if (fDtrace1 \|\| fDtrace2)
11716	{
11717	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11718	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11719	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11720	switch (enmEvent1)
11721	{
11722	/** @todo consider which extra parameters would be helpful for each probe. */
11723	case DBGFEVENT_END: break;
11724	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11725	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11726	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11727	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11728	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11729	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11730	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11731	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11732	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11733	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11734	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11735	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11736	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11737	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11738	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11739	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11740	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11741	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11742	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11743	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11744	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11745	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11746	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11747	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11748	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11749	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11750	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11751	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11752	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11753	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11754	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11755	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11756	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11757	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11758	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11759	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11760	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11761	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11762	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11763	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11764	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11765	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11766	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11767	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11768	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11769	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11770	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11771	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11772	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11773	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11774	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11775	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11776	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11777	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11778	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11779	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11780	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11781	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11782	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11783	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11784	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11785	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11786	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11787	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11788	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11789	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11790	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11791	}
11792	switch (enmEvent2)
11793	{
11794	/** @todo consider which extra parameters would be helpful for each probe. */
11795	case DBGFEVENT_END: break;
11796	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11797	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11798	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11799	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11800	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11801	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11802	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11803	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11804	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11805	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11806	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11807	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11808	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11809	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11810	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11811	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11812	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11813	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11814	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11815	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11816	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11817	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11818	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11819	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11820	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11821	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11822	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11823	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11824	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11825	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11826	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11827	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11828	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11829	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11830	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11831	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11832	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11833	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11834	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11835	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11836	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11837	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11838	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11839	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11840	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11841	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11842	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11843	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11844	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11845	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11846	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11847	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11848	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11849	}
11850	}
11851
11852	/*
11853	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11854	* the DBGF call will do a full check).
11855	*
11856	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11857	* Note! If we have to events, we prioritize the first, i.e. the instruction
11858	* one, in order to avoid event nesting.
11859	*/
11860	PVM pVM = pVCpu->CTX_SUFF(pVM);
11861	if ( enmEvent1 != DBGFEVENT_END
11862	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11863	{
11864	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11865	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11866	if (rcStrict != VINF_SUCCESS)
11867	return rcStrict;
11868	}
11869	else if ( enmEvent2 != DBGFEVENT_END
11870	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11871	{
11872	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11873	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11874	if (rcStrict != VINF_SUCCESS)
11875	return rcStrict;
11876	}
11877
11878	return VINF_SUCCESS;
11879	}
11880
11881
11882	/**
11883	* Single-stepping VM-exit filtering.
11884	*
11885	* This is preprocessing the VM-exits and deciding whether we've gotten far
11886	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11887	* handling is performed.
11888	*
11889	* @returns Strict VBox status code (i.e. informational status codes too).
11890	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11891	* @param pVmxTransient The VMX-transient structure.
11892	* @param pDbgState The debug state.
11893	*/
11894	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11895	{
11896	/*
11897	* Expensive (saves context) generic dtrace VM-exit probe.
11898	*/
11899	uint32_t const uExitReason = pVmxTransient->uExitReason;
11900	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11901	{ /* more likely */ }
11902	else
11903	{
11904	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11905	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11906	AssertRC(rc);
11907	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11908	}
11909
11910	/*
11911	* Check for host NMI, just to get that out of the way.
11912	*/
11913	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11914	{ /* normally likely */ }
11915	else
11916	{
11917	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11918	AssertRCReturn(rc2, rc2);
11919	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11920	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11921	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11922	}
11923
11924	/*
11925	* Check for single stepping event if we're stepping.
11926	*/
11927	if (pVCpu->hm.s.fSingleInstruction)
11928	{
11929	switch (uExitReason)
11930	{
11931	case VMX_EXIT_MTF:
11932	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11933
11934	/* Various events: */
11935	case VMX_EXIT_XCPT_OR_NMI:
11936	case VMX_EXIT_EXT_INT:
11937	case VMX_EXIT_TRIPLE_FAULT:
11938	case VMX_EXIT_INT_WINDOW:
11939	case VMX_EXIT_NMI_WINDOW:
11940	case VMX_EXIT_TASK_SWITCH:
11941	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11942	case VMX_EXIT_APIC_ACCESS:
11943	case VMX_EXIT_EPT_VIOLATION:
11944	case VMX_EXIT_EPT_MISCONFIG:
11945	case VMX_EXIT_PREEMPT_TIMER:
11946
11947	/* Instruction specific VM-exits: */
11948	case VMX_EXIT_CPUID:
11949	case VMX_EXIT_GETSEC:
11950	case VMX_EXIT_HLT:
11951	case VMX_EXIT_INVD:
11952	case VMX_EXIT_INVLPG:
11953	case VMX_EXIT_RDPMC:
11954	case VMX_EXIT_RDTSC:
11955	case VMX_EXIT_RSM:
11956	case VMX_EXIT_VMCALL:
11957	case VMX_EXIT_VMCLEAR:
11958	case VMX_EXIT_VMLAUNCH:
11959	case VMX_EXIT_VMPTRLD:
11960	case VMX_EXIT_VMPTRST:
11961	case VMX_EXIT_VMREAD:
11962	case VMX_EXIT_VMRESUME:
11963	case VMX_EXIT_VMWRITE:
11964	case VMX_EXIT_VMXOFF:
11965	case VMX_EXIT_VMXON:
11966	case VMX_EXIT_MOV_CRX:
11967	case VMX_EXIT_MOV_DRX:
11968	case VMX_EXIT_IO_INSTR:
11969	case VMX_EXIT_RDMSR:
11970	case VMX_EXIT_WRMSR:
11971	case VMX_EXIT_MWAIT:
11972	case VMX_EXIT_MONITOR:
11973	case VMX_EXIT_PAUSE:
11974	case VMX_EXIT_GDTR_IDTR_ACCESS:
11975	case VMX_EXIT_LDTR_TR_ACCESS:
11976	case VMX_EXIT_INVEPT:
11977	case VMX_EXIT_RDTSCP:
11978	case VMX_EXIT_INVVPID:
11979	case VMX_EXIT_WBINVD:
11980	case VMX_EXIT_XSETBV:
11981	case VMX_EXIT_RDRAND:
11982	case VMX_EXIT_INVPCID:
11983	case VMX_EXIT_VMFUNC:
11984	case VMX_EXIT_RDSEED:
11985	case VMX_EXIT_XSAVES:
11986	case VMX_EXIT_XRSTORS:
11987	{
11988	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11989	AssertRCReturn(rc, rc);
11990	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11991	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11992	return VINF_EM_DBG_STEPPED;
11993	break;
11994	}
11995
11996	/* Errors and unexpected events: */
11997	case VMX_EXIT_INIT_SIGNAL:
11998	case VMX_EXIT_SIPI:
11999	case VMX_EXIT_IO_SMI:
12000	case VMX_EXIT_SMI:
12001	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12002	case VMX_EXIT_ERR_MSR_LOAD:
12003	case VMX_EXIT_ERR_MACHINE_CHECK:
12004	case VMX_EXIT_PML_FULL:
12005	case VMX_EXIT_VIRTUALIZED_EOI:
12006	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12007	break;
12008
12009	default:
12010	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12011	break;
12012	}
12013	}
12014
12015	/*
12016	* Check for debugger event breakpoints and dtrace probes.
12017	*/
12018	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12019	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12020	{
12021	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12022	if (rcStrict != VINF_SUCCESS)
12023	return rcStrict;
12024	}
12025
12026	/*
12027	* Normal processing.
12028	*/
12029	#ifdef HMVMX_USE_FUNCTION_TABLE
12030	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12031	#else
12032	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12033	#endif
12034	}
12035
12036
12037	/**
12038	* Single steps guest code using hardware-assisted VMX.
12039	*
12040	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12041	* but single-stepping through the hypervisor debugger.
12042	*
12043	* @returns Strict VBox status code (i.e. informational status codes too).
12044	* @param pVCpu The cross context virtual CPU structure.
12045	* @param pcLoops Pointer to the number of executed loops.
12046	*
12047	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12048	*/
12049	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12050	{
12051	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12052	Assert(pcLoops);
12053	Assert(*pcLoops <= cMaxResumeLoops);
12054
12055	VMXTRANSIENT VmxTransient;
12056	RT_ZERO(VmxTransient);
12057	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12058
12059	/* Set HMCPU indicators. */
12060	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12061	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12062	pVCpu->hm.s.fDebugWantRdTscExit = false;
12063	pVCpu->hm.s.fUsingDebugLoop = true;
12064
12065	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12066	VMXRUNDBGSTATE DbgState;
12067	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12068	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12069
12070	/*
12071	* The loop.
12072	*/
12073	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12074	for (;;)
12075	{
12076	Assert(!HMR0SuspendPending());
12077	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12078	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12079	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12080
12081	/* Set up VM-execution controls the next two can respond to. */
12082	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12083
12084	/*
12085	* Preparatory work for running guest code, this may force us to
12086	* return to ring-3.
12087	*
12088	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12089	*/
12090	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12091	if (rcStrict != VINF_SUCCESS)
12092	break;
12093
12094	/* Interrupts are disabled at this point! */
12095	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12096
12097	/* Override any obnoxious code in the above two calls. */
12098	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12099
12100	/*
12101	* Finally execute the guest.
12102	*/
12103	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12104
12105	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12106	/* Interrupts are re-enabled at this point! */
12107
12108	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12109	if (RT_SUCCESS(rcRun))
12110	{ /* very likely */ }
12111	else
12112	{
12113	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12114	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12115	return rcRun;
12116	}
12117
12118	/* Profile the VM-exit. */
12119	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12120	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12121	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12122	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12123	HMVMX_START_EXIT_DISPATCH_PROF();
12124
12125	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12126
12127	/*
12128	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12129	*/
12130	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12131	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12132	if (rcStrict != VINF_SUCCESS)
12133	break;
12134	if (++(*pcLoops) > cMaxResumeLoops)
12135	{
12136	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12137	rcStrict = VINF_EM_RAW_INTERRUPT;
12138	break;
12139	}
12140
12141	/*
12142	* Stepping: Did the RIP change, if so, consider it a single step.
12143	* Otherwise, make sure one of the TFs gets set.
12144	*/
12145	if (fStepping)
12146	{
12147	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12148	AssertRC(rc);
12149	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12150	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12151	{
12152	rcStrict = VINF_EM_DBG_STEPPED;
12153	break;
12154	}
12155	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12156	}
12157
12158	/*
12159	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12160	*/
12161	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12162	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12163	}
12164
12165	/*
12166	* Clear the X86_EFL_TF if necessary.
12167	*/
12168	if (pVCpu->hm.s.fClearTrapFlag)
12169	{
12170	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12171	AssertRC(rc);
12172	pVCpu->hm.s.fClearTrapFlag = false;
12173	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12174	}
12175	/** @todo there seems to be issues with the resume flag when the monitor trap
12176	* flag is pending without being used. Seen early in bios init when
12177	* accessing APIC page in protected mode. */
12178
12179	/*
12180	* Restore VM-exit control settings as we may not re-enter this function the
12181	* next time around.
12182	*/
12183	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12184
12185	/* Restore HMCPU indicators. */
12186	pVCpu->hm.s.fUsingDebugLoop = false;
12187	pVCpu->hm.s.fDebugWantRdTscExit = false;
12188	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12189
12190	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12191	return rcStrict;
12192	}
12193
12194
12195	/** @} */
12196
12197
12198	/**
12199	* Checks if any expensive dtrace probes are enabled and we should go to the
12200	* debug loop.
12201	*
12202	* @returns true if we should use debug loop, false if not.
12203	*/
12204	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12205	{
12206	/* It's probably faster to OR the raw 32-bit counter variables together.
12207	Since the variables are in an array and the probes are next to one
12208	another (more or less), we have good locality. So, better read
12209	eight-nine cache lines ever time and only have one conditional, than
12210	128+ conditionals, right? */
12211	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12212	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12213	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12214	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12215	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12216	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12217	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12218	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12219	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12220	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12221	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12222	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12223	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12224	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12225	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12226	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12227	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12228	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12229	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12230	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12231	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12232	) != 0
12233	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12234	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12235	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12236	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12237	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12238	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12239	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12240	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12241	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12242	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12243	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12244	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12245	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12246	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12247	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12248	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12249	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12250	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12251	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12252	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12253	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12254	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12255	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12256	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12257	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12258	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12259	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12260	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12261	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12262	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12263	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12264	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12265	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12266	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12267	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12268	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12269	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12270	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12271	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12272	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12273	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12274	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12275	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12276	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12277	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12278	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12279	) != 0
12280	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12281	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12282	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12283	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12284	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12285	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12286	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12287	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12288	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12289	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12290	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12291	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12292	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12293	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12294	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12295	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12296	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12297	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12298	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12299	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12300	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12301	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12302	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12303	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12304	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12305	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12306	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12307	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12308	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12309	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12310	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12311	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12312	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12313	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12314	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12315	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12316	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12317	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12318	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12319	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12320	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12321	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12322	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12323	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12324	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12325	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12326	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12327	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12328	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12329	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12330	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12331	) != 0;
12332	}
12333
12334
12335	/**
12336	* Runs the guest using hardware-assisted VMX.
12337	*
12338	* @returns Strict VBox status code (i.e. informational status codes too).
12339	* @param pVCpu The cross context virtual CPU structure.
12340	*/
12341	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12342	{
12343	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12344	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12345	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12346	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12347
12348	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12349
12350	VBOXSTRICTRC rcStrict;
12351	uint32_t cLoops = 0;
12352	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12353	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12354	#else
12355	bool const fInNestedGuestMode = false;
12356	#endif
12357	if (!fInNestedGuestMode)
12358	{
12359	if ( !pVCpu->hm.s.fUseDebugLoop
12360	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12361	&& !DBGFIsStepping(pVCpu)
12362	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12363	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12364	else
12365	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12366	}
12367	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12368	else
12369	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12370
12371	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12372	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12373	#endif
12374
12375	if (rcStrict == VERR_EM_INTERPRETER)
12376	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12377	else if (rcStrict == VINF_EM_RESET)
12378	rcStrict = VINF_EM_TRIPLE_FAULT;
12379
12380	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12381	if (RT_FAILURE(rc2))
12382	{
12383	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12384	rcStrict = rc2;
12385	}
12386	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12387	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12388	return rcStrict;
12389	}
12390
12391
12392	#ifndef HMVMX_USE_FUNCTION_TABLE
12393	/**
12394	* Handles a guest VM-exit from hardware-assisted VMX execution.
12395	*
12396	* @returns Strict VBox status code (i.e. informational status codes too).
12397	* @param pVCpu The cross context virtual CPU structure.
12398	* @param pVmxTransient The VMX-transient structure.
12399	*/
12400	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12401	{
12402	#ifdef DEBUG_ramshankar
12403	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12404	do { \
12405	if (a_fSave != 0) \
12406	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12407	VBOXSTRICTRC rcStrict = a_CallExpr; \
12408	if (a_fSave != 0) \
12409	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12410	return rcStrict; \
12411	} while (0)
12412	#else
12413	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12414	#endif
12415	uint32_t const rcReason = pVmxTransient->uExitReason;
12416	switch (rcReason)
12417	{
12418	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12419	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12420	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12421	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12422	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12423	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12424	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12425	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12426	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12427	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12428	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12429	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12430	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12431	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12432	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12433	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12434	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12435	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12436	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12437	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12438	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12439	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12440	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12441	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12442	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12443	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12444	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12445	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12446	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12447	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12448	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12449	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12450	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12451	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12452	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12453	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12454	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12455	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12456	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12457	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12458	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12459	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12460	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12461	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12462	#else
12463	case VMX_EXIT_VMCLEAR:
12464	case VMX_EXIT_VMLAUNCH:
12465	case VMX_EXIT_VMPTRLD:
12466	case VMX_EXIT_VMPTRST:
12467	case VMX_EXIT_VMREAD:
12468	case VMX_EXIT_VMRESUME:
12469	case VMX_EXIT_VMWRITE:
12470	case VMX_EXIT_VMXOFF:
12471	case VMX_EXIT_VMXON:
12472	case VMX_EXIT_INVVPID:
12473	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12474	#endif
12475
12476	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12477	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12478	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12479
12480	case VMX_EXIT_RSM:
12481	case VMX_EXIT_RDSEED:
12482	case VMX_EXIT_ENCLS:
12483	case VMX_EXIT_INVEPT:
12484	case VMX_EXIT_VMFUNC:
12485	case VMX_EXIT_XSAVES:
12486	case VMX_EXIT_XRSTORS:
12487	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12488
12489	case VMX_EXIT_INIT_SIGNAL:
12490	case VMX_EXIT_SIPI:
12491	case VMX_EXIT_IO_SMI:
12492	case VMX_EXIT_SMI:
12493	case VMX_EXIT_ERR_MSR_LOAD:
12494	case VMX_EXIT_ERR_MACHINE_CHECK:
12495	case VMX_EXIT_PML_FULL:
12496	case VMX_EXIT_VIRTUALIZED_EOI:
12497	case VMX_EXIT_APIC_WRITE:
12498	default:
12499	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12500	}
12501	#undef VMEXIT_CALL_RET
12502	}
12503	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12504
12505
12506	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12507	/**
12508	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12509	*
12510	* @returns Strict VBox status code (i.e. informational status codes too).
12511	* @param pVCpu The cross context virtual CPU structure.
12512	* @param pVmxTransient The VMX-transient structure.
12513	*/
12514	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12515	{
12516	VBOXSTRICTRC rcStrict;
12517	uint32_t const uExitReason = pVmxTransient->uExitReason;
12518	switch (uExitReason)
12519	{
12520	case VMX_EXIT_EPT_MISCONFIG:
12521	rcStrict = hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12522	break;
12523
12524	case VMX_EXIT_EPT_VIOLATION:
12525	rcStrict = hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12526	break;
12527
12528	case VMX_EXIT_IO_INSTR:
12529	{
12530	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12531	AssertRCReturn(rc, rc);
12532
12533	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12534	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12535	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12536
12537	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
12538	uint8_t const cbAccess = s_aIOSizes[uIOSize];
12539	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
12540	{
12541	/*
12542	* IN/OUT instruction:
12543	* - Provides VM-exit instruction length.
12544	*
12545	* INS/OUTS instruction:
12546	* - Provides VM-exit instruction length.
12547	* - Provides Guest-linear address.
12548	* - Optionally provides VM-exit instruction info (depends on CPU feature).
12549	*/
12550	PVM pVM = pVCpu->CTX_SUFF(pVM);
12551	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12552	AssertRCReturn(rc, rc);
12553
12554	/* Make sure we don't use stale VMX-transient info. */
12555	pVmxTransient->ExitInstrInfo.u = 0;
12556	pVmxTransient->uGuestLinearAddr = 0;
12557
12558	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
12559	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12560	if (fIOString)
12561	{
12562	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12563	if (fVmxInsOutsInfo)
12564	{
12565	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
12566	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12567	}
12568	}
12569	AssertRCReturn(rc, rc);
12570
12571	VMXVEXITINFO ExitInfo;
12572	RT_ZERO(ExitInfo);
12573	ExitInfo.uReason = uExitReason;
12574	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12575	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12576	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12577	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
12578	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12579	}
12580	else
12581	rcStrict = hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
12582	break;
12583	}
12584
12585	case VMX_EXIT_HLT:
12586	{
12587	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12588	AssertRCReturn(rc, rc);
12589	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
12590	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12591	else
12592	rcStrict = hmR0VmxExitHlt(pVCpu, pVmxTransient);
12593	break;
12594	}
12595
12596	case VMX_EXIT_RDTSC:
12597	{
12598	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12599	AssertRCReturn(rc, rc);
12600	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12601	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12602	else
12603	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12604	break;
12605	}
12606
12607	case VMX_EXIT_RDTSCP:
12608	{
12609	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12610	AssertRCReturn(rc, rc);
12611	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12612	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12613	else
12614	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12615	break;
12616	}
12617
12618	/*
12619	* Instructions that cause VM-exits unconditionally.
12620	* - Provides VM-exit instruction length ONLY.
12621	*/
12622	case VMX_EXIT_CPUID:
12623	case VMX_EXIT_VMCALL:
12624	case VMX_EXIT_GETSEC:
12625	case VMX_EXIT_INVD:
12626	case VMX_EXIT_XSETBV:
12627	case VMX_EXIT_VMLAUNCH:
12628	case VMX_EXIT_VMRESUME:
12629	case VMX_EXIT_VMXOFF:
12630	{
12631	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12632	AssertRCReturn(rc, rc);
12633	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12634	break;
12635	}
12636
12637	/*
12638	* Instructions that cause VM-exits unconditionally.
12639	* - Provides VM-exit instruction length.
12640	* - Provides VM-exit information.
12641	* - Optionally provides VM-exit qualification.
12642	*
12643	* Since VM-exit qualification is 0 for all VM-exits where it is not
12644	* applicable, reading and passing it to the guest should produce
12645	* defined behavior.
12646	*
12647	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12648	*/
12649	case VMX_EXIT_INVEPT:
12650	case VMX_EXIT_INVVPID:
12651	case VMX_EXIT_VMCLEAR:
12652	case VMX_EXIT_VMPTRLD:
12653	case VMX_EXIT_VMPTRST:
12654	case VMX_EXIT_VMXON:
12655	{
12656	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12657	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12658	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12659	AssertRCReturn(rc, rc);
12660
12661	VMXVEXITINFO ExitInfo;
12662	RT_ZERO(ExitInfo);
12663	ExitInfo.uReason = uExitReason;
12664	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12665	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12666	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12667	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12668	break;
12669	}
12670
12671	case VMX_EXIT_INVLPG:
12672	{
12673	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12674	{
12675	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12676	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12677	AssertRCReturn(rc, rc);
12678
12679	VMXVEXITINFO ExitInfo;
12680	RT_ZERO(ExitInfo);
12681	ExitInfo.uReason = uExitReason;
12682	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12683	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12684	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12685	}
12686	else
12687	rcStrict = hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
12688	break;
12689	}
12690
12691	case VMX_EXIT_INVPCID:
12692	{
12693	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12694	{
12695	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12696	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12697	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12698	AssertRCReturn(rc, rc);
12699
12700	VMXVEXITINFO ExitInfo;
12701	RT_ZERO(ExitInfo);
12702	ExitInfo.uReason = uExitReason;
12703	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12704	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12705	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12706	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12707	}
12708	else
12709	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12710	break;
12711	}
12712
12713	case VMX_EXIT_RDMSR:
12714	{
12715	uint32_t fMsrpm;
12716	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12717	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12718	else
12719	fMsrpm = VMXMSRPM_EXIT_RD;
12720
12721	if (fMsrpm & VMXMSRPM_EXIT_RD)
12722	{
12723	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12724	AssertRCReturn(rc, rc);
12725	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12726	}
12727	else
12728	rcStrict = hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
12729	break;
12730	}
12731
12732	case VMX_EXIT_WRMSR:
12733	{
12734	uint32_t fMsrpm;
12735	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12736	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12737	else
12738	fMsrpm = VMXMSRPM_EXIT_WR;
12739
12740	if (fMsrpm & VMXMSRPM_EXIT_WR)
12741	{
12742	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12743	AssertRCReturn(rc, rc);
12744	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12745	}
12746	else
12747	rcStrict = hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
12748	break;
12749	}
12750
12751	case VMX_EXIT_TASK_SWITCH:
12752	{
12753	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12754	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12755	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12756	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12757	AssertRCReturn(rc, rc);
12758
12759	VMXVEXITINFO ExitInfo;
12760	RT_ZERO(ExitInfo);
12761	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12762	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12763
12764	VMXVEXITEVENTINFO ExitEventInfo;
12765	RT_ZERO(ExitInfo);
12766	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
12767	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
12768
12769	rcStrict = IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
12770	break;
12771	}
12772
12773	case VMX_EXIT_WBINVD:
12774	{
12775	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
12776	{
12777	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12778	AssertRCReturn(rc, rc);
12779	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12780	}
12781	else
12782	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12783	break;
12784	}
12785
12786	case VMX_EXIT_MTF:
12787	{
12788	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
12789	rcStrict = IEMExecVmxVmexit(pVCpu, uExitReason);
12790	break;
12791	}
12792
12793	case VMX_EXIT_APIC_ACCESS:
12794	case VMX_EXIT_XCPT_OR_NMI:
12795	case VMX_EXIT_MOV_CRX:
12796
12797	case VMX_EXIT_EXT_INT:
12798	{
12799	/* We shouldn't direct physical interrupts to the nested-guest. */
12800	rcStrict = hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12801	break;
12802	}
12803
12804	case VMX_EXIT_INT_WINDOW:
12805	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12806	case VMX_EXIT_MWAIT:
12807	case VMX_EXIT_MONITOR:
12808
12809	case VMX_EXIT_PAUSE:
12810	{
12811	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
12812	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
12813	* PAUSE when executing a nested-guest? If it does not, we would not need
12814	* to check for the intercepts here. Just call VM-exit... */
12815	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
12816	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
12817	{
12818	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12819	AssertRCReturn(rc, rc);
12820	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12821	}
12822	else
12823	rcStrict = hmR0VmxExitPause(pVCpu, pVmxTransient);
12824	break;
12825	}
12826
12827	case VMX_EXIT_PREEMPT_TIMER:
12828	case VMX_EXIT_MOV_DRX:
12829	case VMX_EXIT_GDTR_IDTR_ACCESS:
12830	case VMX_EXIT_LDTR_TR_ACCESS:
12831	case VMX_EXIT_RDRAND:
12832	case VMX_EXIT_RDPMC:
12833	case VMX_EXIT_VMREAD:
12834	case VMX_EXIT_VMWRITE:
12835	case VMX_EXIT_RSM:
12836	case VMX_EXIT_RDSEED:
12837	case VMX_EXIT_ENCLS:
12838	case VMX_EXIT_VMFUNC:
12839	case VMX_EXIT_XSAVES:
12840	case VMX_EXIT_XRSTORS:
12841
12842	case VMX_EXIT_TRIPLE_FAULT:
12843	case VMX_EXIT_NMI_WINDOW:
12844	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12845
12846	case VMX_EXIT_INIT_SIGNAL:
12847	case VMX_EXIT_SIPI:
12848	case VMX_EXIT_IO_SMI:
12849	case VMX_EXIT_SMI:
12850	case VMX_EXIT_ERR_MSR_LOAD:
12851	case VMX_EXIT_ERR_MACHINE_CHECK:
12852	case VMX_EXIT_PML_FULL:
12853	case VMX_EXIT_VIRTUALIZED_EOI:
12854	case VMX_EXIT_APIC_WRITE:
12855	default:
12856	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12857	break;
12858	}
12859
12860	return rcStrict;
12861	}
12862	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12863
12864
12865	#ifdef VBOX_STRICT
12866	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12867	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12868	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12869
12870	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12871	do { \
12872	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12873	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12874	} while (0)
12875
12876	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12877	do { \
12878	AssertPtr((a_pVCpu)); \
12879	AssertPtr((a_pVmxTransient)); \
12880	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12881	Assert((a_pVmxTransient)->pVmcsInfo); \
12882	Assert(ASMIntAreEnabled()); \
12883	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12884	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12885	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12886	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12887	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12888	HMVMX_ASSERT_PREEMPT_CPUID(); \
12889	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12890	} while (0)
12891
12892	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12893	do { \
12894	Log4Func(("\n")); \
12895	} while (0)
12896	#else
12897	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12898	do { \
12899	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12900	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12901	} while (0)
12902	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12903	#endif
12904
12905
12906	/**
12907	* Advances the guest RIP by the specified number of bytes.
12908	*
12909	* @param pVCpu The cross context virtual CPU structure.
12910	* @param cbInstr Number of bytes to advance the RIP by.
12911	*
12912	* @remarks No-long-jump zone!!!
12913	*/
12914	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12915	{
12916	/* Advance the RIP. */
12917	pVCpu->cpum.GstCtx.rip += cbInstr;
12918	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12919
12920	/* Update interrupt inhibition. */
12921	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12922	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12923	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12924	}
12925
12926
12927	/**
12928	* Advances the guest RIP after reading it from the VMCS.
12929	*
12930	* @returns VBox status code, no informational status codes.
12931	* @param pVCpu The cross context virtual CPU structure.
12932	* @param pVmxTransient The VMX-transient structure.
12933	*
12934	* @remarks No-long-jump zone!!!
12935	*/
12936	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12937	{
12938	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12939	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12940	AssertRCReturn(rc, rc);
12941
12942	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12943	return VINF_SUCCESS;
12944	}
12945
12946
12947	/**
12948	* Handle a condition that occurred while delivering an event through the guest
12949	* IDT.
12950	*
12951	* @returns Strict VBox status code (i.e. informational status codes too).
12952	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12953	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12954	* to continue execution of the guest which will delivery the \#DF.
12955	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12956	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12957	*
12958	* @param pVCpu The cross context virtual CPU structure.
12959	* @param pVmxTransient The VMX-transient structure.
12960	*
12961	* @remarks No-long-jump zone!!!
12962	*/
12963	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12964	{
12965	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12966
12967	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12968	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12969	AssertRCReturn(rc2, rc2);
12970
12971	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12972	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12973	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12974	{
12975	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12976	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12977
12978	/*
12979	* If the event was a software interrupt (generated with INT n) or a software exception
12980	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12981	* can handle the VM-exit and continue guest execution which will re-execute the
12982	* instruction rather than re-injecting the exception, as that can cause premature
12983	* trips to ring-3 before injection and involve TRPM which currently has no way of
12984	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12985	* the problem).
12986	*/
12987	IEMXCPTRAISE enmRaise;
12988	IEMXCPTRAISEINFO fRaiseInfo;
12989	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12990	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12991	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12992	{
12993	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12994	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12995	}
12996	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12997	{
12998	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12999	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13000	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13001	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13002	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13003	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13004
13005	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13006
13007	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13008	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13009	{
13010	pVmxTransient->fVectoringPF = true;
13011	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13012	}
13013	}
13014	else
13015	{
13016	/*
13017	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13018	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13019	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13020	*/
13021	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13022	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13023	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13024	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13025	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13026	}
13027
13028	/*
13029	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13030	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13031	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13032	* subsequent VM-entry would fail.
13033	*
13034	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13035	*/
13036	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
13037	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13038	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13039	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13040	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13041	{
13042	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
13043	}
13044
13045	switch (enmRaise)
13046	{
13047	case IEMXCPTRAISE_CURRENT_XCPT:
13048	{
13049	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13050	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13051	Assert(rcStrict == VINF_SUCCESS);
13052	break;
13053	}
13054
13055	case IEMXCPTRAISE_PREV_EVENT:
13056	{
13057	uint32_t u32ErrCode;
13058	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13059	{
13060	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13061	AssertRCReturn(rc2, rc2);
13062	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13063	}
13064	else
13065	u32ErrCode = 0;
13066
13067	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13068	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13069	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13070	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13071
13072	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13073	pVCpu->hm.s.Event.u32ErrCode));
13074	Assert(rcStrict == VINF_SUCCESS);
13075	break;
13076	}
13077
13078	case IEMXCPTRAISE_REEXEC_INSTR:
13079	Assert(rcStrict == VINF_SUCCESS);
13080	break;
13081
13082	case IEMXCPTRAISE_DOUBLE_FAULT:
13083	{
13084	/*
13085	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13086	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13087	*/
13088	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13089	{
13090	pVmxTransient->fVectoringDoublePF = true;
13091	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13092	pVCpu->cpum.GstCtx.cr2));
13093	rcStrict = VINF_SUCCESS;
13094	}
13095	else
13096	{
13097	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13098	hmR0VmxSetPendingXcptDF(pVCpu);
13099	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13100	uIdtVector, uExitVector));
13101	rcStrict = VINF_HM_DOUBLE_FAULT;
13102	}
13103	break;
13104	}
13105
13106	case IEMXCPTRAISE_TRIPLE_FAULT:
13107	{
13108	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13109	rcStrict = VINF_EM_RESET;
13110	break;
13111	}
13112
13113	case IEMXCPTRAISE_CPU_HANG:
13114	{
13115	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13116	rcStrict = VERR_EM_GUEST_CPU_HANG;
13117	break;
13118	}
13119
13120	default:
13121	{
13122	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13123	rcStrict = VERR_VMX_IPE_2;
13124	break;
13125	}
13126	}
13127	}
13128	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13129	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13130	&& uExitVector != X86_XCPT_DF
13131	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13132	{
13133	/*
13134	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13135	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13136	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13137	*/
13138	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
13139	{
13140	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
13141	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
13142	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
13143	}
13144	}
13145
13146	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13147	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13148	return rcStrict;
13149	}
13150
13151
13152	/** @name VM-exit handlers.
13153	* @{
13154	*/
13155	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13156	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13157	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13158
13159	/**
13160	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13161	*/
13162	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13163	{
13164	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13165	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13166	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13167	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13168	return VINF_SUCCESS;
13169	return VINF_EM_RAW_INTERRUPT;
13170	}
13171
13172
13173	/**
13174	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13175	*/
13176	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13177	{
13178	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13179	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13180
13181	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13182	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13183	AssertRCReturn(rc, rc);
13184
13185	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13186	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13187	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13188	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13189
13190	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13191	{
13192	/*
13193	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13194	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13195	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13196	*
13197	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13198	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13199	*/
13200	VMXDispatchHostNmi();
13201	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
13202	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13203	return VINF_SUCCESS;
13204	}
13205
13206	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13207	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13208	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
13209	{ /* likely */ }
13210	else
13211	{
13212	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
13213	rcStrictRc1 = VINF_SUCCESS;
13214	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13215	return rcStrictRc1;
13216	}
13217
13218	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13219	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13220	switch (uIntType)
13221	{
13222	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13223	Assert(uVector == X86_XCPT_DB);
13224	RT_FALL_THRU();
13225	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13226	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13227	RT_FALL_THRU();
13228	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13229	{
13230	/*
13231	* If there's any exception caused as a result of event injection, the resulting
13232	* secondary/final execption will be pending, we shall continue guest execution
13233	* after injecting the event. The page-fault case is complicated and we manually
13234	* handle any currently pending event in hmR0VmxExitXcptPF.
13235	*/
13236	if (!pVCpu->hm.s.Event.fPending)
13237	{ /* likely */ }
13238	else if (uVector != X86_XCPT_PF)
13239	{
13240	rc = VINF_SUCCESS;
13241	break;
13242	}
13243
13244	switch (uVector)
13245	{
13246	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13247	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13248	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13249	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13250	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13251	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13252
13253	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13254	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13255	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13256	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13257	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13258	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13259	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13260	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13261	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13262	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13263	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13264	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13265	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13266	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13267	default:
13268	{
13269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13270	if (pVmcsInfo->RealMode.fRealOnV86Active)
13271	{
13272	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13273	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13274	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13275
13276	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13277	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13278	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13279	AssertRCReturn(rc, rc);
13280	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13281	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13282	0 /* GCPtrFaultAddress */);
13283	}
13284	else
13285	{
13286	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13287	pVCpu->hm.s.u32HMError = uVector;
13288	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
13289	}
13290	break;
13291	}
13292	}
13293	break;
13294	}
13295
13296	default:
13297	{
13298	pVCpu->hm.s.u32HMError = uExitIntInfo;
13299	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13300	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13301	break;
13302	}
13303	}
13304	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13305	return rc;
13306	}
13307
13308
13309	/**
13310	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13311	*/
13312	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13313	{
13314	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13315
13316	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13317	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13318	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13319	AssertRCReturn(rc, rc);
13320
13321	/* Evaluate and deliver pending events and resume guest execution. */
13322	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13323	return VINF_SUCCESS;
13324	}
13325
13326
13327	/**
13328	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13329	*/
13330	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13331	{
13332	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13333
13334	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13335	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13336	{
13337	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13338	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13339	}
13340
13341	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13342
13343	/*
13344	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13345	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13346	*/
13347	uint32_t fIntrState;
13348	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13349	AssertRCReturn(rc, rc);
13350	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13351	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13352	{
13353	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13354	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13355
13356	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13357	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13358	AssertRCReturn(rc, rc);
13359	}
13360
13361	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13362	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13363	AssertRCReturn(rc, rc);
13364
13365	/* Evaluate and deliver pending events and resume guest execution. */
13366	return VINF_SUCCESS;
13367	}
13368
13369
13370	/**
13371	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13372	*/
13373	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13374	{
13375	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13376	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13377	}
13378
13379
13380	/**
13381	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13382	*/
13383	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13384	{
13385	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13386	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13387	}
13388
13389
13390	/**
13391	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13392	*/
13393	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13394	{
13395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13396
13397	/*
13398	* Get the state we need and update the exit history entry.
13399	*/
13400	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13401	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13402	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13403	AssertRCReturn(rc, rc);
13404
13405	VBOXSTRICTRC rcStrict;
13406	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13407	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13408	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13409	if (!pExitRec)
13410	{
13411	/*
13412	* Regular CPUID instruction execution.
13413	*/
13414	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13415	if (rcStrict == VINF_SUCCESS)
13416	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13417	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13418	{
13419	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13420	rcStrict = VINF_SUCCESS;
13421	}
13422	}
13423	else
13424	{
13425	/*
13426	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13427	*/
13428	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13429	AssertRCReturn(rc2, rc2);
13430
13431	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13432	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13433
13434	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13435	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13436
13437	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13438	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13439	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13440	}
13441	return rcStrict;
13442	}
13443
13444
13445	/**
13446	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13447	*/
13448	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13449	{
13450	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13451
13452	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13453	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13454	AssertRCReturn(rc, rc);
13455
13456	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13457	return VINF_EM_RAW_EMULATE_INSTR;
13458
13459	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13460	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13461	}
13462
13463
13464	/**
13465	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13466	*/
13467	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13468	{
13469	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13470
13471	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13472	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13473	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13474	AssertRCReturn(rc, rc);
13475
13476	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13477	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13478	{
13479	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13480	we must reset offsetting on VM-entry. See @bugref{6634}. */
13481	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13482	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13483	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13484	}
13485	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13486	{
13487	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13488	rcStrict = VINF_SUCCESS;
13489	}
13490	return rcStrict;
13491	}
13492
13493
13494	/**
13495	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13496	*/
13497	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13498	{
13499	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13500
13501	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13502	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13503	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13504	AssertRCReturn(rc, rc);
13505
13506	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13507	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13508	{
13509	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13510	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13511	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13512	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13513	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13514	}
13515	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13516	{
13517	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13518	rcStrict = VINF_SUCCESS;
13519	}
13520	return rcStrict;
13521	}
13522
13523
13524	/**
13525	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13526	*/
13527	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13528	{
13529	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13530
13531	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13532	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13533	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13534	AssertRCReturn(rc, rc);
13535
13536	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13537	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13538	if (RT_LIKELY(rc == VINF_SUCCESS))
13539	{
13540	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13541	Assert(pVmxTransient->cbInstr == 2);
13542	}
13543	else
13544	{
13545	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13546	rc = VERR_EM_INTERPRETER;
13547	}
13548	return rc;
13549	}
13550
13551
13552	/**
13553	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13554	*/
13555	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13556	{
13557	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13558
13559	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13560	if (EMAreHypercallInstructionsEnabled(pVCpu))
13561	{
13562	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13563	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13564	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13565	AssertRCReturn(rc, rc);
13566
13567	/* Perform the hypercall. */
13568	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13569	if (rcStrict == VINF_SUCCESS)
13570	{
13571	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13572	AssertRCReturn(rc, rc);
13573	}
13574	else
13575	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13576	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13577	\|\| RT_FAILURE(rcStrict));
13578
13579	/* If the hypercall changes anything other than guest's general-purpose registers,
13580	we would need to reload the guest changed bits here before VM-entry. */
13581	}
13582	else
13583	Log4Func(("Hypercalls not enabled\n"));
13584
13585	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13586	if (RT_FAILURE(rcStrict))
13587	{
13588	hmR0VmxSetPendingXcptUD(pVCpu);
13589	rcStrict = VINF_SUCCESS;
13590	}
13591
13592	return rcStrict;
13593	}
13594
13595
13596	/**
13597	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13598	*/
13599	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13600	{
13601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13602	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13603
13604	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13605	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13606	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13607	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13608	AssertRCReturn(rc, rc);
13609
13610	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13611
13612	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13613	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13614	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13615	{
13616	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13617	rcStrict = VINF_SUCCESS;
13618	}
13619	else
13620	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13621	VBOXSTRICTRC_VAL(rcStrict)));
13622	return rcStrict;
13623	}
13624
13625
13626	/**
13627	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13628	*/
13629	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13630	{
13631	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13632
13633	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13634	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13635	AssertRCReturn(rc, rc);
13636
13637	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13638	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13639	if (RT_LIKELY(rc == VINF_SUCCESS))
13640	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13641	else
13642	{
13643	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13644	rc = VERR_EM_INTERPRETER;
13645	}
13646	return rc;
13647	}
13648
13649
13650	/**
13651	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13652	*/
13653	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13654	{
13655	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13656
13657	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13658	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13659	AssertRCReturn(rc, rc);
13660
13661	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13662	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13663	rc = VBOXSTRICTRC_VAL(rc2);
13664	if (RT_LIKELY( rc == VINF_SUCCESS
13665	\|\| rc == VINF_EM_HALT))
13666	{
13667	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13668	AssertRCReturn(rc3, rc3);
13669
13670	if ( rc == VINF_EM_HALT
13671	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13672	rc = VINF_SUCCESS;
13673	}
13674	else
13675	{
13676	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13677	rc = VERR_EM_INTERPRETER;
13678	}
13679	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13680	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13681	return rc;
13682	}
13683
13684
13685	/**
13686	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13687	* VM-exit.
13688	*/
13689	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13690	{
13691	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13692	return VINF_EM_RESET;
13693	}
13694
13695
13696	/**
13697	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13698	*/
13699	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13700	{
13701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13702
13703	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13704	AssertRCReturn(rc, rc);
13705
13706	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13707	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13708	rc = VINF_SUCCESS;
13709	else
13710	rc = VINF_EM_HALT;
13711
13712	if (rc != VINF_SUCCESS)
13713	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13714	return rc;
13715	}
13716
13717
13718	/**
13719	* VM-exit handler for instructions that result in a \#UD exception delivered to
13720	* the guest.
13721	*/
13722	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13723	{
13724	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13725	hmR0VmxSetPendingXcptUD(pVCpu);
13726	return VINF_SUCCESS;
13727	}
13728
13729
13730	/**
13731	* VM-exit handler for expiry of the VMX-preemption timer.
13732	*/
13733	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13734	{
13735	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13736
13737	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13738	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13739
13740	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13741	PVM pVM = pVCpu->CTX_SUFF(pVM);
13742	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13743	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13744	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13745	}
13746
13747
13748	/**
13749	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13750	*/
13751	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13752	{
13753	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13754
13755	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13756	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13757	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13758	AssertRCReturn(rc, rc);
13759
13760	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13761	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13762	: HM_CHANGED_RAISED_XCPT_MASK);
13763
13764	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13765	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13766
13767	return rcStrict;
13768	}
13769
13770
13771	/**
13772	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13773	*/
13774	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13775	{
13776	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13777	/** @todo Use VM-exit instruction information. */
13778	return VERR_EM_INTERPRETER;
13779	}
13780
13781
13782	/**
13783	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13784	* Error VM-exit.
13785	*/
13786	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13787	{
13788	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13789	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13790	AssertRCReturn(rc, rc);
13791
13792	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13793	if (RT_FAILURE(rc))
13794	return rc;
13795
13796	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13797	NOREF(uInvalidReason);
13798
13799	#ifdef VBOX_STRICT
13800	uint32_t fIntrState;
13801	RTHCUINTREG uHCReg;
13802	uint64_t u64Val;
13803	uint32_t u32Val;
13804	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13805	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13806	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13807	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13808	AssertRCReturn(rc, rc);
13809
13810	Log4(("uInvalidReason %u\n", uInvalidReason));
13811	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13812	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13813	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13814	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13815
13816	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13817	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13818	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13819	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13820	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13821	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13822	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13823	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13824	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13825	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13826	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13827	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13828
13829	hmR0DumpRegs(pVCpu);
13830	#endif
13831
13832	return VERR_VMX_INVALID_GUEST_STATE;
13833	}
13834
13835	/**
13836	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
13837	*/
13838	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13839	{
13840	/*
13841	* Cummulative notes of all recognized but unexpected VM-exits.
13842	* This does -not- cover those VM-exits like a page-fault occurring when say nested-paging
13843	* is used.
13844	*
13845	* VMX_EXIT_INIT_SIGNAL:
13846	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13847	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
13848	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
13849	*
13850	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
13851	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
13852	* See Intel spec. "23.8 Restrictions on VMX operation".
13853	*
13854	* VMX_EXIT_SIPI:
13855	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
13856	* activity state is used. We don't make use of it as our guests don't have direct
13857	* access to the host local APIC.
13858	*
13859	* See Intel spec. 25.3 "Other Causes of VM-exits".
13860	*
13861	* VMX_EXIT_IO_SMI:
13862	* VMX_EXIT_SMI:
13863	* This can only happen if we support dual-monitor treatment of SMI, which can be
13864	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
13865	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
13866	* VMX root mode or receive an SMI. If we get here, something funny is going on.
13867	*
13868	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13869	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13870	*
13871	* VMX_EXIT_ERR_MACHINE_CHECK:
13872	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
13873	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
13874	* #MC exception abort class exception is raised. We thus cannot assume a
13875	* reasonable chance of continuing any sort of execution and we bail.
13876	*
13877	* See Intel spec. 15.1 "Machine-check Architecture".
13878	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
13879	*
13880	* VMX_EXIT_ERR_MSR_LOAD:
13881	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
13882	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
13883	* execution.
13884	*
13885	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
13886	*
13887	* VMX_EXIT_PML_FULL:
13888	* VMX_EXIT_VIRTUALIZED_EOI:
13889	* VMX_EXIT_APIC_WRITE:
13890	* We do not currently support any of these features and thus they are all unexpected
13891	* VM-exits.
13892	*/
13893	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13894	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
13895	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13896	}
13897
13898
13899	/**
13900	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13901	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13902	* Conditional VM-exit.
13903	*/
13904	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13905	{
13906	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13907
13908	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13910	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13911	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13912	return VERR_EM_INTERPRETER;
13913	AssertMsgFailed(("Unexpected XDTR access\n"));
13914	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13915	}
13916
13917
13918	/**
13919	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13920	*/
13921	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13922	{
13923	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13924
13925	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13926	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13927	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13928	return VERR_EM_INTERPRETER;
13929	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13930	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13931	}
13932
13933
13934	/**
13935	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13936	*/
13937	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13938	{
13939	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13940
13941	/** @todo Optimize this: We currently drag in in the whole MSR state
13942	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13943	* MSRs required. That would require changes to IEM and possibly CPUM too.
13944	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13945	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13946	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13947	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13948	switch (idMsr)
13949	{
13950	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13951	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13952	}
13953
13954	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13955	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13956	AssertRCReturn(rc, rc);
13957
13958	Log4Func(("ecx=%#RX32\n", idMsr));
13959
13960	#ifdef VBOX_STRICT
13961	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13962	{
13963	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13964	&& idMsr != MSR_K6_EFER)
13965	{
13966	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13967	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13968	}
13969	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13970	{
13971	Assert(pVmcsInfo->pvMsrBitmap);
13972	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13973	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13974	{
13975	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13976	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13977	}
13978	}
13979	}
13980	#endif
13981
13982	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13984	if (rcStrict == VINF_SUCCESS)
13985	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13986	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13987	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13988	{
13989	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13990	rcStrict = VINF_SUCCESS;
13991	}
13992	else
13993	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13994
13995	return rcStrict;
13996	}
13997
13998
13999	/**
14000	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14001	*/
14002	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14003	{
14004	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14005
14006	/** @todo Optimize this: We currently drag in in the whole MSR state
14007	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14008	* MSRs required. That would require changes to IEM and possibly CPUM too.
14009	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14010	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14011	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14012
14013	/*
14014	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14015	* Although we don't need to fetch the base as it will be overwritten shortly, while
14016	* loading guest-state we would also load the entire segment register including limit
14017	* and attributes and thus we need to load them here.
14018	*/
14019	switch (idMsr)
14020	{
14021	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14022	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14023	}
14024
14025	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14026	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14027	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14028	AssertRCReturn(rc, rc);
14029
14030	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14031
14032	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14034
14035	if (rcStrict == VINF_SUCCESS)
14036	{
14037	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14038
14039	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14040	if ( idMsr == MSR_IA32_APICBASE
14041	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14042	&& idMsr <= MSR_IA32_X2APIC_END))
14043	{
14044	/*
14045	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14046	* When full APIC register virtualization is implemented we'll have to make
14047	* sure APIC state is saved from the VMCS before IEM changes it.
14048	*/
14049	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14050	}
14051	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14052	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14053	else if (idMsr == MSR_K6_EFER)
14054	{
14055	/*
14056	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14057	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14058	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14059	*/
14060	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14061	}
14062
14063	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14064	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14065	{
14066	switch (idMsr)
14067	{
14068	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14069	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14070	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14071	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14072	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14073	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14074	default:
14075	{
14076	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14077	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14078	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14079	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14080	break;
14081	}
14082	}
14083	}
14084	#ifdef VBOX_STRICT
14085	else
14086	{
14087	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14088	switch (idMsr)
14089	{
14090	case MSR_IA32_SYSENTER_CS:
14091	case MSR_IA32_SYSENTER_EIP:
14092	case MSR_IA32_SYSENTER_ESP:
14093	case MSR_K8_FS_BASE:
14094	case MSR_K8_GS_BASE:
14095	{
14096	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14097	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14098	}
14099
14100	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14101	default:
14102	{
14103	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14104	{
14105	/* EFER MSR writes are always intercepted. */
14106	if (idMsr != MSR_K6_EFER)
14107	{
14108	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14109	idMsr));
14110	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14111	}
14112	}
14113
14114	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14115	{
14116	Assert(pVmcsInfo->pvMsrBitmap);
14117	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14118	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14119	{
14120	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14121	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14122	}
14123	}
14124	break;
14125	}
14126	}
14127	}
14128	#endif /* VBOX_STRICT */
14129	}
14130	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14131	{
14132	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14133	rcStrict = VINF_SUCCESS;
14134	}
14135	else
14136	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14137
14138	return rcStrict;
14139	}
14140
14141
14142	/**
14143	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14144	*/
14145	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14146	{
14147	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14148	/** @todo The guest has likely hit a contended spinlock. We might want to
14149	* poke a schedule different guest VCPU. */
14150	return VINF_EM_RAW_INTERRUPT;
14151	}
14152
14153
14154	/**
14155	* VM-exit handler for when the TPR value is lowered below the specified
14156	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14157	*/
14158	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14159	{
14160	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14161	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14162
14163	/*
14164	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14165	* We'll re-evaluate pending interrupts and inject them before the next VM
14166	* entry so we can just continue execution here.
14167	*/
14168	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14169	return VINF_SUCCESS;
14170	}
14171
14172
14173	/**
14174	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14175	* VM-exit.
14176	*
14177	* @retval VINF_SUCCESS when guest execution can continue.
14178	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14179	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
14180	* interpreter.
14181	*/
14182	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14183	{
14184	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14185	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14186
14187	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14188	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14189	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14190	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14191	AssertRCReturn(rc, rc);
14192
14193	VBOXSTRICTRC rcStrict;
14194	PVM pVM = pVCpu->CTX_SUFF(pVM);
14195	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
14196	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14197	switch (uAccessType)
14198	{
14199	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14200	{
14201	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14202	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14203	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
14204	AssertMsg( rcStrict == VINF_SUCCESS
14205	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14206	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14207
14208	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14209	{
14210	case 0:
14211	{
14212	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14213	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14214	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14215	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14216
14217	/*
14218	* This is a kludge for handling switches back to real mode when we try to use
14219	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14220	* deal with special selector values, so we have to return to ring-3 and run
14221	* there till the selector values are V86 mode compatible.
14222	*
14223	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14224	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
14225	* at the end of this function.
14226	*/
14227	if ( rc == VINF_SUCCESS
14228	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
14229	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14230	&& (uOldCr0 & X86_CR0_PE)
14231	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
14232	{
14233	/** @todo check selectors rather than returning all the time. */
14234	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14235	rcStrict = VINF_EM_RESCHEDULE_REM;
14236	}
14237	break;
14238	}
14239
14240	case 2:
14241	{
14242	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14243	/* Nothing to do here, CR2 it's not part of the VMCS. */
14244	break;
14245	}
14246
14247	case 3:
14248	{
14249	Assert( !pVM->hm.s.fNestedPaging
14250	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14251	\|\| pVCpu->hm.s.fUsingDebugLoop);
14252	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14253	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14254	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14255	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14256	break;
14257	}
14258
14259	case 4:
14260	{
14261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14262	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14263	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14264	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14265	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14266	break;
14267	}
14268
14269	case 8:
14270	{
14271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14272	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14273	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14274	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14275	break;
14276	}
14277	default:
14278	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
14279	break;
14280	}
14281	break;
14282	}
14283
14284	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14285	{
14286	Assert( !pVM->hm.s.fNestedPaging
14287	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14288	\|\| pVCpu->hm.s.fUsingDebugLoop
14289	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
14290	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14291	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
14292	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14293
14294	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
14295	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
14296	AssertMsg( rcStrict == VINF_SUCCESS
14297	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14298	#ifdef VBOX_WITH_STATISTICS
14299	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14300	{
14301	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14302	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14303	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14304	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14305	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14306	}
14307	#endif
14308	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14309	VBOXSTRICTRC_VAL(rcStrict)));
14310	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14311	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14312	else
14313	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14314	break;
14315	}
14316
14317	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14318	{
14319	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14320	AssertMsg( rcStrict == VINF_SUCCESS
14321	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14322
14323	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14325	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14326	break;
14327	}
14328
14329	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14330	{
14331	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14332	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14333	AssertRCReturn(rc, rc);
14334	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14335	pVmxTransient->uGuestLinearAddr);
14336	AssertMsg( rcStrict == VINF_SUCCESS
14337	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14338	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14339
14340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14341	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14342	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14343	break;
14344	}
14345
14346	default:
14347	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14348	VERR_VMX_UNEXPECTED_EXCEPTION);
14349	}
14350
14351	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14352	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14353	if (rcStrict == VINF_IEM_RAISED_XCPT)
14354	{
14355	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14356	rcStrict = VINF_SUCCESS;
14357	}
14358
14359	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14360	NOREF(pVM);
14361	return rcStrict;
14362	}
14363
14364
14365	/**
14366	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14367	* VM-exit.
14368	*/
14369	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14370	{
14371	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14372	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14373
14374	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14375	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14376	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14377	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14378	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14379	\| CPUMCTX_EXTRN_EFER);
14380	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14381	AssertRCReturn(rc, rc);
14382
14383	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14384	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14385	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14386	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14387	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14388	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14389	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14390	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14391
14392	/*
14393	* Update exit history to see if this exit can be optimized.
14394	*/
14395	VBOXSTRICTRC rcStrict;
14396	PCEMEXITREC pExitRec = NULL;
14397	if ( !fGstStepping
14398	&& !fDbgStepping)
14399	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14400	!fIOString
14401	? !fIOWrite
14402	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14403	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14404	: !fIOWrite
14405	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14406	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14407	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14408	if (!pExitRec)
14409	{
14410	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14411	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14412
14413	uint32_t const cbValue = s_aIOSizes[uIOSize];
14414	uint32_t const cbInstr = pVmxTransient->cbInstr;
14415	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14416	PVM pVM = pVCpu->CTX_SUFF(pVM);
14417	if (fIOString)
14418	{
14419	/*
14420	* INS/OUTS - I/O String instruction.
14421	*
14422	* Use instruction-information if available, otherwise fall back on
14423	* interpreting the instruction.
14424	*/
14425	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14426	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14427	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14428	if (fInsOutsInfo)
14429	{
14430	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14431	AssertRCReturn(rc2, rc2);
14432	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14433	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14434	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14435	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14436	if (fIOWrite)
14437	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14438	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14439	else
14440	{
14441	/*
14442	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14443	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14444	* See Intel Instruction spec. for "INS".
14445	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14446	*/
14447	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14448	}
14449	}
14450	else
14451	rcStrict = IEMExecOne(pVCpu);
14452
14453	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14454	fUpdateRipAlready = true;
14455	}
14456	else
14457	{
14458	/*
14459	* IN/OUT - I/O instruction.
14460	*/
14461	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14462	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14463	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14464	if (fIOWrite)
14465	{
14466	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14467	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14468	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14469	&& !pCtx->eflags.Bits.u1TF)
14470	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14471	}
14472	else
14473	{
14474	uint32_t u32Result = 0;
14475	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14476	if (IOM_SUCCESS(rcStrict))
14477	{
14478	/* Save result of I/O IN instr. in AL/AX/EAX. */
14479	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14480	}
14481	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14482	&& !pCtx->eflags.Bits.u1TF)
14483	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14485	}
14486	}
14487
14488	if (IOM_SUCCESS(rcStrict))
14489	{
14490	if (!fUpdateRipAlready)
14491	{
14492	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14494	}
14495
14496	/*
14497	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14498	* while booting Fedora 17 64-bit guest.
14499	*
14500	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14501	*/
14502	if (fIOString)
14503	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14504
14505	/*
14506	* If any I/O breakpoints are armed, we need to check if one triggered
14507	* and take appropriate action.
14508	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14509	*/
14510	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14511	AssertRCReturn(rc, rc);
14512
14513	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14514	* execution engines about whether hyper BPs and such are pending. */
14515	uint32_t const uDr7 = pCtx->dr[7];
14516	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14517	&& X86_DR7_ANY_RW_IO(uDr7)
14518	&& (pCtx->cr4 & X86_CR4_DE))
14519	\|\| DBGFBpIsHwIoArmed(pVM)))
14520	{
14521	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14522
14523	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14524	VMMRZCallRing3Disable(pVCpu);
14525	HM_DISABLE_PREEMPT(pVCpu);
14526
14527	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14528
14529	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14530	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14531	{
14532	/* Raise #DB. */
14533	if (fIsGuestDbgActive)
14534	ASMSetDR6(pCtx->dr[6]);
14535	if (pCtx->dr[7] != uDr7)
14536	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14537
14538	hmR0VmxSetPendingXcptDB(pVCpu);
14539	}
14540	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14541	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14542	else if ( rcStrict2 != VINF_SUCCESS
14543	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14544	rcStrict = rcStrict2;
14545	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14546
14547	HM_RESTORE_PREEMPT();
14548	VMMRZCallRing3Enable(pVCpu);
14549	}
14550	}
14551
14552	#ifdef VBOX_STRICT
14553	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14554	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14555	Assert(!fIOWrite);
14556	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14557	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14558	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14559	Assert(fIOWrite);
14560	else
14561	{
14562	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14563	* statuses, that the VMM device and some others may return. See
14564	* IOM_SUCCESS() for guidance. */
14565	AssertMsg( RT_FAILURE(rcStrict)
14566	\|\| rcStrict == VINF_SUCCESS
14567	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14568	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14569	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14570	\|\| rcStrict == VINF_EM_RAW_TO_R3
14571	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14572	# endif
14573	}
14574	#endif
14575	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14576	}
14577	else
14578	{
14579	/*
14580	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14581	*/
14582	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14583	AssertRCReturn(rc2, rc2);
14584	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14585	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14586	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14587	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14588	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14589	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14590
14591	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14592	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14593
14594	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14595	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14596	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14597	}
14598	return rcStrict;
14599	}
14600
14601
14602	/**
14603	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14604	* VM-exit.
14605	*/
14606	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14607	{
14608	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14609
14610	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14611	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14612	AssertRCReturn(rc, rc);
14613	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14614	{
14615	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14616	AssertRCReturn(rc, rc);
14617	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14618	{
14619	uint32_t uErrCode;
14620	RTGCUINTPTR GCPtrFaultAddress;
14621	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14622	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14623	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14624	if (fErrorCodeValid)
14625	{
14626	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14627	AssertRCReturn(rc, rc);
14628	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14629	}
14630	else
14631	uErrCode = 0;
14632
14633	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14634	&& uVector == X86_XCPT_PF)
14635	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14636	else
14637	GCPtrFaultAddress = 0;
14638
14639	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14640	AssertRCReturn(rc, rc);
14641
14642	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14643	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14644
14645	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14647	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14648	}
14649	}
14650
14651	/* Fall back to the interpreter to emulate the task-switch. */
14652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14653	return VERR_EM_INTERPRETER;
14654	}
14655
14656
14657	/**
14658	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14659	*/
14660	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14661	{
14662	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14663
14664	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14665	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14666	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14667	AssertRCReturn(rc, rc);
14668	return VINF_EM_DBG_STEPPED;
14669	}
14670
14671
14672	/**
14673	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14674	*/
14675	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14676	{
14677	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14679
14680	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14681	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14682	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14683	{
14684	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14685	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14686	{
14687	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14688	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14689	}
14690	}
14691	else
14692	{
14693	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14694	rcStrict1 = VINF_SUCCESS;
14695	return rcStrict1;
14696	}
14697
14698	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14699	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14700	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14701	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14702	AssertRCReturn(rc, rc);
14703
14704	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14705	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14706	VBOXSTRICTRC rcStrict2;
14707	switch (uAccessType)
14708	{
14709	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14710	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14711	{
14712	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14713	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14714	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14715
14716	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14717	GCPhys &= PAGE_BASE_GC_MASK;
14718	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14719	PVM pVM = pVCpu->CTX_SUFF(pVM);
14720	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14721	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14722
14723	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14724	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14725	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14726	CPUMCTX2CORE(pCtx), GCPhys);
14727	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14728	if ( rcStrict2 == VINF_SUCCESS
14729	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14730	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14731	{
14732	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14733	\| HM_CHANGED_GUEST_APIC_TPR);
14734	rcStrict2 = VINF_SUCCESS;
14735	}
14736	break;
14737	}
14738
14739	default:
14740	Log4Func(("uAccessType=%#x\n", uAccessType));
14741	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14742	break;
14743	}
14744
14745	if (rcStrict2 != VINF_SUCCESS)
14746	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14747	return rcStrict2;
14748	}
14749
14750
14751	/**
14752	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14753	* VM-exit.
14754	*/
14755	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14756	{
14757	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14758
14759	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14760	if (pVmxTransient->fWasGuestDebugStateActive)
14761	{
14762	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14763	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14764	}
14765
14766	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14767	if ( !pVCpu->hm.s.fSingleInstruction
14768	&& !pVmxTransient->fWasHyperDebugStateActive)
14769	{
14770	Assert(!DBGFIsStepping(pVCpu));
14771	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14772
14773	/* Don't intercept MOV DRx any more. */
14774	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14775	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14776	AssertRCReturn(rc, rc);
14777
14778	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14779	VMMRZCallRing3Disable(pVCpu);
14780	HM_DISABLE_PREEMPT(pVCpu);
14781
14782	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14783	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14784	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14785
14786	HM_RESTORE_PREEMPT();
14787	VMMRZCallRing3Enable(pVCpu);
14788
14789	#ifdef VBOX_WITH_STATISTICS
14790	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14791	AssertRCReturn(rc, rc);
14792	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14793	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14794	else
14795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14796	#endif
14797	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14798	return VINF_SUCCESS;
14799	}
14800
14801	/*
14802	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14803	* The EFER MSR is always up-to-date.
14804	* Update the segment registers and DR7 from the CPU.
14805	*/
14806	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14807	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14808	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14809	AssertRCReturn(rc, rc);
14810	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14811
14812	PVM pVM = pVCpu->CTX_SUFF(pVM);
14813	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14814	{
14815	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14816	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14817	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14818	if (RT_SUCCESS(rc))
14819	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14821	}
14822	else
14823	{
14824	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14825	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14826	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14827	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14828	}
14829
14830	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14831	if (RT_SUCCESS(rc))
14832	{
14833	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14834	AssertRCReturn(rc2, rc2);
14835	return VINF_SUCCESS;
14836	}
14837	return rc;
14838	}
14839
14840
14841	/**
14842	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14843	* Conditional VM-exit.
14844	*/
14845	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14846	{
14847	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14848	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14849
14850	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14851	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14852	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14853	{
14854	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14855	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14856	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14857	{
14858	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14859	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14860	}
14861	}
14862	else
14863	{
14864	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14865	rcStrict1 = VINF_SUCCESS;
14866	return rcStrict1;
14867	}
14868
14869	/*
14870	* Get sufficent state and update the exit history entry.
14871	*/
14872	RTGCPHYS GCPhys;
14873	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14874	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14875	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14876	AssertRCReturn(rc, rc);
14877
14878	VBOXSTRICTRC rcStrict;
14879	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14880	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14881	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14882	if (!pExitRec)
14883	{
14884	/*
14885	* If we succeed, resume guest execution.
14886	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14887	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14888	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14889	* weird case. See @bugref{6043}.
14890	*/
14891	PVM pVM = pVCpu->CTX_SUFF(pVM);
14892	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14893	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14894	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14895	if ( rcStrict == VINF_SUCCESS
14896	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14897	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14898	{
14899	/* Successfully handled MMIO operation. */
14900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14901	\| HM_CHANGED_GUEST_APIC_TPR);
14902	rcStrict = VINF_SUCCESS;
14903	}
14904	}
14905	else
14906	{
14907	/*
14908	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14909	*/
14910	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14911	AssertRCReturn(rc2, rc2);
14912
14913	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14914	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14915
14916	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14917	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14918
14919	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14920	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14921	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14922	}
14923	return VBOXSTRICTRC_TODO(rcStrict);
14924	}
14925
14926
14927	/**
14928	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14929	* VM-exit.
14930	*/
14931	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14932	{
14933	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14934	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14935
14936	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14937	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14938	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14939	{
14940	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14941	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14942	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14943	}
14944	else
14945	{
14946	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14947	rcStrict1 = VINF_SUCCESS;
14948	return rcStrict1;
14949	}
14950
14951	RTGCPHYS GCPhys;
14952	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14953	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14954	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14955	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14956	AssertRCReturn(rc, rc);
14957
14958	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14959	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14960
14961	RTGCUINT uErrorCode = 0;
14962	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14963	uErrorCode \|= X86_TRAP_PF_ID;
14964	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14965	uErrorCode \|= X86_TRAP_PF_RW;
14966	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14967	uErrorCode \|= X86_TRAP_PF_P;
14968
14969	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14970
14971
14972	/* Handle the pagefault trap for the nested shadow table. */
14973	PVM pVM = pVCpu->CTX_SUFF(pVM);
14974	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14975
14976	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14977	pCtx->cs.Sel, pCtx->rip));
14978
14979	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14980	TRPMResetTrap(pVCpu);
14981
14982	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14983	if ( rcStrict2 == VINF_SUCCESS
14984	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14985	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14986	{
14987	/* Successfully synced our nested page tables. */
14988	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14989	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14990	return VINF_SUCCESS;
14991	}
14992
14993	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14994	return rcStrict2;
14995	}
14996
14997	/** @} */
14998
14999	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15000	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
15001	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15002
15003	/**
15004	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
15005	*/
15006	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15007	{
15008	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15010
15011	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15012	AssertRCReturn(rc, rc);
15013
15014	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15015	{
15016	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15017	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15018
15019	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15020	* provides VM-exit instruction length. If this causes problem later,
15021	* disassemble the instruction like it's done on AMD-V. */
15022	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15023	AssertRCReturn(rc2, rc2);
15024	return rc;
15025	}
15026
15027	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15028	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15029	return rc;
15030	}
15031
15032
15033	/**
15034	* VM-exit exception handler for \#BP (Breakpoint exception).
15035	*/
15036	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15037	{
15038	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15039	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15040
15041	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15042	AssertRCReturn(rc, rc);
15043
15044	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15045	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15046	if (rc == VINF_EM_RAW_GUEST_TRAP)
15047	{
15048	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15049	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15050	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15051	AssertRCReturn(rc, rc);
15052
15053	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15054	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15055	}
15056
15057	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15058	return rc;
15059	}
15060
15061
15062	/**
15063	* VM-exit exception handler for \#AC (alignment check exception).
15064	*/
15065	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15066	{
15067	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15068
15069	/*
15070	* Re-inject it. We'll detect any nesting before getting here.
15071	*/
15072	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15073	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15074	AssertRCReturn(rc, rc);
15075	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15076
15077	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15078	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15079	return VINF_SUCCESS;
15080	}
15081
15082
15083	/**
15084	* VM-exit exception handler for \#DB (Debug exception).
15085	*/
15086	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15087	{
15088	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15090
15091	/*
15092	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
15093	* for processing.
15094	*/
15095	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15096
15097	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15098	uint64_t const uDR6 = X86_DR6_INIT_VAL
15099	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15100	\| X86_DR6_BD \| X86_DR6_BS));
15101
15102	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15103	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15104	Log6Func(("rc=%Rrc\n", rc));
15105	if (rc == VINF_EM_RAW_GUEST_TRAP)
15106	{
15107	/*
15108	* The exception was for the guest. Update DR6, DR7.GD and
15109	* IA32_DEBUGCTL.LBR before forwarding it.
15110	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15111	*/
15112	VMMRZCallRing3Disable(pVCpu);
15113	HM_DISABLE_PREEMPT(pVCpu);
15114
15115	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15116	pCtx->dr[6] \|= uDR6;
15117	if (CPUMIsGuestDebugStateActive(pVCpu))
15118	ASMSetDR6(pCtx->dr[6]);
15119
15120	HM_RESTORE_PREEMPT();
15121	VMMRZCallRing3Enable(pVCpu);
15122
15123	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15124	AssertRCReturn(rc, rc);
15125
15126	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15127	pCtx->dr[7] &= ~X86_DR7_GD;
15128
15129	/* Paranoia. */
15130	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15131	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15132
15133	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15134	AssertRCReturn(rc, rc);
15135
15136	/*
15137	* Raise #DB in the guest.
15138	*
15139	* It is important to reflect exactly what the VM-exit gave us (preserving the
15140	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15141	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15142	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15143	*
15144	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15145	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15146	*/
15147	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15148	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15149	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15150	AssertRCReturn(rc, rc);
15151	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15152	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15153	return VINF_SUCCESS;
15154	}
15155
15156	/*
15157	* Not a guest trap, must be a hypervisor related debug event then.
15158	* Update DR6 in case someone is interested in it.
15159	*/
15160	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15161	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15162	CPUMSetHyperDR6(pVCpu, uDR6);
15163
15164	return rc;
15165	}
15166
15167
15168	/**
15169	* Hacks its way around the lovely mesa driver's backdoor accesses.
15170	*
15171	* @sa hmR0SvmHandleMesaDrvGp.
15172	*/
15173	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15174	{
15175	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15176	RT_NOREF(pCtx);
15177
15178	/* For now we'll just skip the instruction. */
15179	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15180	}
15181
15182
15183	/**
15184	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15185	* backdoor logging w/o checking what it is running inside.
15186	*
15187	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15188	* backdoor port and magic numbers loaded in registers.
15189	*
15190	* @returns true if it is, false if it isn't.
15191	* @sa hmR0SvmIsMesaDrvGp.
15192	*/
15193	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15194	{
15195	/* 0xed: IN eAX,dx */
15196	uint8_t abInstr[1];
15197	if (pVmxTransient->cbInstr != sizeof(abInstr))
15198	return false;
15199
15200	/* Check that it is #GP(0). */
15201	if (pVmxTransient->uExitIntErrorCode != 0)
15202	return false;
15203
15204	/* Check magic and port. */
15205	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15206	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15207	if (pCtx->rax != UINT32_C(0x564d5868))
15208	return false;
15209	if (pCtx->dx != UINT32_C(0x5658))
15210	return false;
15211
15212	/* Flat ring-3 CS. */
15213	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15214	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15215	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15216	if (pCtx->cs.Attr.n.u2Dpl != 3)
15217	return false;
15218	if (pCtx->cs.u64Base != 0)
15219	return false;
15220
15221	/* Check opcode. */
15222	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15223	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15224	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15225	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15226	if (RT_FAILURE(rc))
15227	return false;
15228	if (abInstr[0] != 0xed)
15229	return false;
15230
15231	return true;
15232	}
15233
15234
15235	/**
15236	* VM-exit exception handler for \#GP (General-protection exception).
15237	*
15238	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15239	*/
15240	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15241	{
15242	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15243	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15244
15245	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15246	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15247	if (pVmcsInfo->RealMode.fRealOnV86Active)
15248	{ /* likely */ }
15249	else
15250	{
15251	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15252	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15253	#endif
15254	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15255	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15256	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15257	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15258	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15259	AssertRCReturn(rc, rc);
15260	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15261	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15262
15263	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15264	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15265	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15266	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15267	else
15268	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15269	return rc;
15270	}
15271
15272	Assert(CPUMIsGuestInRealModeEx(pCtx));
15273	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15274
15275	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15276	AssertRCReturn(rc, rc);
15277
15278	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15279	if (rcStrict == VINF_SUCCESS)
15280	{
15281	if (!CPUMIsGuestInRealModeEx(pCtx))
15282	{
15283	/*
15284	* The guest is no longer in real-mode, check if we can continue executing the
15285	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15286	*/
15287	pVmcsInfo->RealMode.fRealOnV86Active = false;
15288	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15289	{
15290	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15291	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15292	}
15293	else
15294	{
15295	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15296	rcStrict = VINF_EM_RESCHEDULE;
15297	}
15298	}
15299	else
15300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15301	}
15302	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15303	{
15304	rcStrict = VINF_SUCCESS;
15305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15306	}
15307	return VBOXSTRICTRC_VAL(rcStrict);
15308	}
15309
15310
15311	/**
15312	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15313	* the exception reported in the VMX transient structure back into the VM.
15314	*
15315	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15316	* up-to-date.
15317	*/
15318	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15319	{
15320	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15321	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15322	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15323	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15324	("uVector=%#x u32XcptBitmap=%#X32\n",
15325	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15326	NOREF(pVmcsInfo);
15327	#endif
15328
15329	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15330	hmR0VmxCheckExitDueToEventDelivery(). */
15331	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15332	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15333	AssertRCReturn(rc, rc);
15334	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15335
15336	#ifdef DEBUG_ramshankar
15337	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15338	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15339	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15340	#endif
15341
15342	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15343	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15344	return VINF_SUCCESS;
15345	}
15346
15347
15348	/**
15349	* VM-exit exception handler for \#PF (Page-fault exception).
15350	*/
15351	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15352	{
15353	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15354	PVM pVM = pVCpu->CTX_SUFF(pVM);
15355	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15356	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15357	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15358	AssertRCReturn(rc, rc);
15359
15360	if (!pVM->hm.s.fNestedPaging)
15361	{ /* likely */ }
15362	else
15363	{
15364	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15365	Assert(pVCpu->hm.s.fUsingDebugLoop);
15366	#endif
15367	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15368	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15369	{
15370	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15371	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15372	}
15373	else
15374	{
15375	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15376	hmR0VmxSetPendingXcptDF(pVCpu);
15377	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15378	}
15379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15380	return rc;
15381	}
15382
15383	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15384	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15385	if (pVmxTransient->fVectoringPF)
15386	{
15387	Assert(pVCpu->hm.s.Event.fPending);
15388	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15389	}
15390
15391	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15392	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15393	AssertRCReturn(rc, rc);
15394
15395	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15396	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15397
15398	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15399	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15400
15401	Log4Func(("#PF: rc=%Rrc\n", rc));
15402	if (rc == VINF_SUCCESS)
15403	{
15404	/*
15405	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15406	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15407	*/
15408	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15409	TRPMResetTrap(pVCpu);
15410	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15411	return rc;
15412	}
15413
15414	if (rc == VINF_EM_RAW_GUEST_TRAP)
15415	{
15416	if (!pVmxTransient->fVectoringDoublePF)
15417	{
15418	/* It's a guest page fault and needs to be reflected to the guest. */
15419	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15420	TRPMResetTrap(pVCpu);
15421	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15422	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15423	uGstErrorCode, pVmxTransient->uExitQual);
15424	}
15425	else
15426	{
15427	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15428	TRPMResetTrap(pVCpu);
15429	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15430	hmR0VmxSetPendingXcptDF(pVCpu);
15431	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15432	}
15433
15434	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15435	return VINF_SUCCESS;
15436	}
15437
15438	TRPMResetTrap(pVCpu);
15439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15440	return rc;
15441	}
15442
15443	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15444	/** @name VMX instruction handlers.
15445	* @{
15446	*/
15447	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15448	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15449	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15450
15451	/**
15452	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15453	*/
15454	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15455	{
15456	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15457
15458	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15459	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15460	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15461	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15462	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15463	AssertRCReturn(rc, rc);
15464
15465	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15466
15467	VMXVEXITINFO ExitInfo;
15468	RT_ZERO(ExitInfo);
15469	ExitInfo.uReason = pVmxTransient->uExitReason;
15470	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15471	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15472	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15473	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15474
15475	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15476	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15477	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15478	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15479	{
15480	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15481	rcStrict = VINF_SUCCESS;
15482	}
15483	return rcStrict;
15484	}
15485
15486
15487	/**
15488	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15489	*/
15490	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15491	{
15492	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15493
15494	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15495	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15496	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15497	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15498	AssertRCReturn(rc, rc);
15499
15500	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15501
15502	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15503	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15504	{
15505	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15506	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15507	}
15508	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15509	return rcStrict;
15510	}
15511
15512
15513	/**
15514	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15515	*/
15516	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15517	{
15518	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15519
15520	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15521	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15522	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15523	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15524	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15525	AssertRCReturn(rc, rc);
15526
15527	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15528
15529	VMXVEXITINFO ExitInfo;
15530	RT_ZERO(ExitInfo);
15531	ExitInfo.uReason = pVmxTransient->uExitReason;
15532	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15533	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15534	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15535	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15536
15537	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15538	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15539	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15540	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15541	{
15542	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15543	rcStrict = VINF_SUCCESS;
15544	}
15545	return rcStrict;
15546	}
15547
15548
15549	/**
15550	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15551	*/
15552	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15553	{
15554	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15555
15556	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15557	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15558	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15559	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15560	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15561	AssertRCReturn(rc, rc);
15562
15563	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15564
15565	VMXVEXITINFO ExitInfo;
15566	RT_ZERO(ExitInfo);
15567	ExitInfo.uReason = pVmxTransient->uExitReason;
15568	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15569	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15570	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15571	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15572
15573	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15574	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15575	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15576	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15577	{
15578	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15579	rcStrict = VINF_SUCCESS;
15580	}
15581	return rcStrict;
15582	}
15583
15584
15585	/**
15586	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15587	*/
15588	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15589	{
15590	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15591
15592	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15593	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15594	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15595	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15596	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15597	AssertRCReturn(rc, rc);
15598
15599	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15600
15601	VMXVEXITINFO ExitInfo;
15602	RT_ZERO(ExitInfo);
15603	ExitInfo.uReason = pVmxTransient->uExitReason;
15604	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15605	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15606	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15607	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15608	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15609
15610	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15611	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15612	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15613	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15614	{
15615	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15616	rcStrict = VINF_SUCCESS;
15617	}
15618	return rcStrict;
15619	}
15620
15621
15622	/**
15623	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15624	*/
15625	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15626	{
15627	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15628
15629	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15630	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15631	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15632	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15633	AssertRCReturn(rc, rc);
15634
15635	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15636
15637	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15638	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15639	{
15640	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15641	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15642	}
15643	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15644	return rcStrict;
15645	}
15646
15647
15648	/**
15649	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15650	*/
15651	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15652	{
15653	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15654
15655	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15656	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15657	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15658	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15659	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15660	AssertRCReturn(rc, rc);
15661
15662	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15663
15664	VMXVEXITINFO ExitInfo;
15665	RT_ZERO(ExitInfo);
15666	ExitInfo.uReason = pVmxTransient->uExitReason;
15667	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15668	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15669	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15670	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15671	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15672
15673	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15674	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15675	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15676	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15677	{
15678	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15679	rcStrict = VINF_SUCCESS;
15680	}
15681	return rcStrict;
15682	}
15683
15684
15685	/**
15686	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15687	*/
15688	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15689	{
15690	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15691
15692	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15693	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15694	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15695	AssertRCReturn(rc, rc);
15696
15697	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15698
15699	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15700	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15701	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15702	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15703	{
15704	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15705	rcStrict = VINF_SUCCESS;
15706	}
15707	return rcStrict;
15708	}
15709
15710
15711	/**
15712	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15713	*/
15714	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15715	{
15716	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15717
15718	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15719	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15720	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15721	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15722	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15723	AssertRCReturn(rc, rc);
15724
15725	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15726
15727	VMXVEXITINFO ExitInfo;
15728	RT_ZERO(ExitInfo);
15729	ExitInfo.uReason = pVmxTransient->uExitReason;
15730	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15731	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15732	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15733	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15734
15735	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15736	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15737	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15738	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15739	{
15740	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15741	rcStrict = VINF_SUCCESS;
15742	}
15743	return rcStrict;
15744	}
15745
15746
15747	/**
15748	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15749	*/
15750	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15751	{
15752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15753
15754	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15755	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15756	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15757	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15758	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15759	AssertRCReturn(rc, rc);
15760
15761	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15762
15763	VMXVEXITINFO ExitInfo;
15764	RT_ZERO(ExitInfo);
15765	ExitInfo.uReason = pVmxTransient->uExitReason;
15766	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15767	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15768	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15769	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15770
15771	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15772	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15773	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15774	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15775	{
15776	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15777	rcStrict = VINF_SUCCESS;
15778	}
15779	return rcStrict;
15780	}
15781
15782	/** @} */
15783	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15784

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78680

Download in other formats: