HMVMXR0.cpp@ 78722

Last change on this file since 78722 was 78720, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Readability.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 666.4 KB

Line
1	/* $Id: HMVMXR0.cpp 78720 2019-05-24 11:34:54Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
366	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369	static FNVMXEXITHANDLER hmR0VmxExitHlt;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
371	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
376	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
377	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
378	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
379	static FNVMXEXITHANDLER hmR0VmxExitVmread;
380	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
381	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
382	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
383	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
384	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
385	#endif
386	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
387	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
388	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
389	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
390	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
391	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
392	static FNVMXEXITHANDLER hmR0VmxExitMwait;
393	static FNVMXEXITHANDLER hmR0VmxExitMtf;
394	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395	static FNVMXEXITHANDLER hmR0VmxExitPause;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
404	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
410	/** @} */
411
412	/** @name Helpers for hardware exceptions VM-exit handlers.
413	* @{
414	*/
415	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
416	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
417	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
418	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
419	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
420	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
421	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	/** @} */
423
424
425	/*********************************************************************************************************************************
426	* Global Variables *
427	*********************************************************************************************************************************/
428	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
429	static const uint32_t g_aVmcsCacheSegBase[] =
430	{
431	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
432	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
433	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
434	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
435	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
436	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
437	};
438	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
439	#endif
440	static const uint32_t g_aVmcsSegBase[] =
441	{
442	VMX_VMCS_GUEST_ES_BASE,
443	VMX_VMCS_GUEST_CS_BASE,
444	VMX_VMCS_GUEST_SS_BASE,
445	VMX_VMCS_GUEST_DS_BASE,
446	VMX_VMCS_GUEST_FS_BASE,
447	VMX_VMCS_GUEST_GS_BASE
448	};
449	static const uint32_t g_aVmcsSegSel[] =
450	{
451	VMX_VMCS16_GUEST_ES_SEL,
452	VMX_VMCS16_GUEST_CS_SEL,
453	VMX_VMCS16_GUEST_SS_SEL,
454	VMX_VMCS16_GUEST_DS_SEL,
455	VMX_VMCS16_GUEST_FS_SEL,
456	VMX_VMCS16_GUEST_GS_SEL
457	};
458	static const uint32_t g_aVmcsSegLimit[] =
459	{
460	VMX_VMCS32_GUEST_ES_LIMIT,
461	VMX_VMCS32_GUEST_CS_LIMIT,
462	VMX_VMCS32_GUEST_SS_LIMIT,
463	VMX_VMCS32_GUEST_DS_LIMIT,
464	VMX_VMCS32_GUEST_FS_LIMIT,
465	VMX_VMCS32_GUEST_GS_LIMIT
466	};
467	static const uint32_t g_aVmcsSegAttr[] =
468	{
469	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
470	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
471	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
472	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
473	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
474	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
475	};
476	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
477	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
478	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
479	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
480
481	#ifdef HMVMX_USE_FUNCTION_TABLE
482	/**
483	* VMX_EXIT dispatch table.
484	*/
485	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
486	{
487	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
488	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
489	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
490	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
491	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
492	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
493	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
494	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
495	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
496	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
497	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
498	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
499	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
500	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
501	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
502	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
503	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
504	/* 17 VMX_EXIT_RSM */ hmR0VmxExitSetPendingXcptUD,
505	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
506	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
507	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
508	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
509	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
510	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
511	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
512	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
513	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
514	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
515	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
516	#else
517	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
518	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
519	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
520	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
521	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
522	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
523	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
524	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
525	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
526	#endif
527	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
528	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
529	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
530	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
531	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
532	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
533	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
534	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
535	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
536	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
537	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
538	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
539	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
540	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
541	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
542	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
543	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
544	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
545	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
546	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
547	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
548	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
549	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
550	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
551	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
552	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
553	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
554	#else
555	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
556	#endif
557	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
558	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
559	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
560	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
561	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
562	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
563	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitSetPendingXcptUD,
564	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitSetPendingXcptUD,
565	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
566	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
567	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
568	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
569	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
570	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitSetPendingXcptUD,
571	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitSetPendingXcptUD,
572	};
573	#endif /* HMVMX_USE_FUNCTION_TABLE */
574
575	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
576	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
577	{
578	/* 0 */ "(Not Used)",
579	/* 1 */ "VMCALL executed in VMX root operation.",
580	/* 2 */ "VMCLEAR with invalid physical address.",
581	/* 3 */ "VMCLEAR with VMXON pointer.",
582	/* 4 */ "VMLAUNCH with non-clear VMCS.",
583	/* 5 */ "VMRESUME with non-launched VMCS.",
584	/* 6 */ "VMRESUME after VMXOFF",
585	/* 7 */ "VM-entry with invalid control fields.",
586	/* 8 */ "VM-entry with invalid host state fields.",
587	/* 9 */ "VMPTRLD with invalid physical address.",
588	/* 10 */ "VMPTRLD with VMXON pointer.",
589	/* 11 */ "VMPTRLD with incorrect revision identifier.",
590	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
591	/* 13 */ "VMWRITE to read-only VMCS component.",
592	/* 14 */ "(Not Used)",
593	/* 15 */ "VMXON executed in VMX root operation.",
594	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
595	/* 17 */ "VM-entry with non-launched executing VMCS.",
596	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
597	/* 19 */ "VMCALL with non-clear VMCS.",
598	/* 20 */ "VMCALL with invalid VM-exit control fields.",
599	/* 21 */ "(Not Used)",
600	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
601	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
602	/* 24 */ "VMCALL with invalid SMM-monitor features.",
603	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
604	/* 26 */ "VM-entry with events blocked by MOV SS.",
605	/* 27 */ "(Not Used)",
606	/* 28 */ "Invalid operand to INVEPT/INVVPID."
607	};
608	#endif /* VBOX_STRICT */
609
610
611	/**
612	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
613	*
614	* Any bit set in this mask is owned by the host/hypervisor and would cause a
615	* VM-exit when modified by the guest.
616	*
617	* @returns The static CR0 guest/host mask.
618	* @param pVCpu The cross context virtual CPU structure.
619	*/
620	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
621	{
622	/*
623	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
624	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
625	*/
626	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
627	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
628	* and @bugref{6944}. */
629	PVM pVM = pVCpu->CTX_SUFF(pVM);
630	return ( X86_CR0_PE
631	\| X86_CR0_NE
632	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
633	\| X86_CR0_PG
634	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
635	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
636	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
637	}
638
639
640	/**
641	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
642	*
643	* Any bit set in this mask is owned by the host/hypervisor and would cause a
644	* VM-exit when modified by the guest.
645	*
646	* @returns The static CR4 guest/host mask.
647	* @param pVCpu The cross context virtual CPU structure.
648	*/
649	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
650	{
651	/*
652	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
653	* these bits are reserved on hardware that does not support them. Since the
654	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
655	* these bits and handle it depending on whether we expose them to the guest.
656	*/
657	PVM pVM = pVCpu->CTX_SUFF(pVM);
658	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
659	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
660	return ( X86_CR4_VMXE
661	\| X86_CR4_VME
662	\| X86_CR4_PAE
663	\| X86_CR4_PGE
664	\| X86_CR4_PSE
665	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
666	\| (fPcid ? X86_CR4_PCIDE : 0));
667	}
668
669
670	/**
671	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
672	* area.
673	*
674	* @returns @c true if it's different, @c false otherwise.
675	* @param pVmcsInfo The VMCS info. object.
676	*/
677	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
678	{
679	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
680	&& pVmcsInfo->pvGuestMsrStore);
681	}
682
683
684	/**
685	* Adds one or more exceptions to the exception bitmap and commits it to the current
686	* VMCS.
687	*
688	* @returns VBox status code.
689	* @param pVmxTransient The VMX-transient structure.
690	* @param uXcptMask The exception(s) to add.
691	*/
692	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
693	{
694	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
695	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
696	if ((uXcptBitmap & uXcptMask) != uXcptMask)
697	{
698	uXcptBitmap \|= uXcptMask;
699	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
700	AssertRCReturn(rc, rc);
701	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
702	}
703	return VINF_SUCCESS;
704	}
705
706
707	/**
708	* Adds an exception to the exception bitmap and commits it to the current VMCS.
709	*
710	* @returns VBox status code.
711	* @param pVmxTransient The VMX-transient structure.
712	* @param uXcpt The exception to add.
713	*/
714	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
715	{
716	Assert(uXcpt <= X86_XCPT_LAST);
717	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
718	}
719
720
721	/**
722	* Remove one or more exceptions from the exception bitmap and commits it to the
723	* current VMCS.
724	*
725	* This takes care of not removing the exception intercept if a nested-guest
726	* requires the exception to be intercepted.
727	*
728	* @returns VBox status code.
729	* @param pVCpu The cross context virtual CPU structure.
730	* @param pVmxTransient The VMX-transient structure.
731	* @param uXcptMask The exception(s) to remove.
732	*/
733	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
734	{
735	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
736	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
737	if (u32XcptBitmap & uXcptMask)
738	{
739	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
740	if (!pVmxTransient->fIsNestedGuest)
741	{ /* likely */ }
742	else
743	{
744	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
745	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
746	}
747	#endif
748	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
749	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
750	\| RT_BIT(X86_XCPT_DE)
751	\| RT_BIT(X86_XCPT_NM)
752	\| RT_BIT(X86_XCPT_TS)
753	\| RT_BIT(X86_XCPT_UD)
754	\| RT_BIT(X86_XCPT_NP)
755	\| RT_BIT(X86_XCPT_SS)
756	\| RT_BIT(X86_XCPT_GP)
757	\| RT_BIT(X86_XCPT_PF)
758	\| RT_BIT(X86_XCPT_MF));
759	#elif defined(HMVMX_ALWAYS_TRAP_PF)
760	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
761	#endif
762	if (uXcptMask)
763	{
764	/* Validate we are not removing any essential exception intercepts. */
765	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
766	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
767	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
768
769	/* Remove it from the exception bitmap. */
770	u32XcptBitmap &= ~uXcptMask;
771
772	/* Commit and update the cache if necessary. */
773	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
774	{
775	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
776	AssertRCReturn(rc, rc);
777	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
778	}
779	}
780	}
781	return VINF_SUCCESS;
782	}
783
784
785	/**
786	* Remove an exceptions from the exception bitmap and commits it to the current
787	* VMCS.
788	*
789	* @returns VBox status code.
790	* @param pVCpu The cross context virtual CPU structure.
791	* @param pVmxTransient The VMX-transient structure.
792	* @param uXcpt The exception to remove.
793	*/
794	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
795	{
796	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
797	}
798
799
800	/**
801	* Loads the VMCS specified by the VMCS info. object.
802	*
803	* @returns VBox status code.
804	* @param pVmcsInfo The VMCS info. object.
805	*/
806	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
807	{
808	Assert(pVmcsInfo);
809	Assert(pVmcsInfo->HCPhysVmcs);
810	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
811
812	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
813	{
814	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
815	if (RT_SUCCESS(rc))
816	{
817	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
818	return VINF_SUCCESS;
819	}
820	return rc;
821	}
822	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
823	}
824
825
826	/**
827	* Clears the VMCS specified by the VMCS info. object.
828	*
829	* @returns VBox status code.
830	* @param pVmcsInfo The VMCS info. object.
831	*/
832	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
833	{
834	Assert(pVmcsInfo);
835	Assert(pVmcsInfo->HCPhysVmcs);
836	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
837
838	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
839	if (RT_SUCCESS(rc))
840	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
841	return rc;
842	}
843
844
845	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
846	/**
847	* Switches the current VMCS to the one specified.
848	*
849	* @returns VBox status code.
850	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
851	* @param pVmcsInfoTo The VMCS info. object we are switching to.
852	*
853	* @remarks Called with interrupts disabled.
854	*/
855	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
856	{
857	Assert(pVmcsInfoFrom);
858	Assert(pVmcsInfoTo);
859
860	/*
861	* Clear the VMCS we are switching out if it has not already been cleared.
862	* This will sync any CPU internal data back to the VMCS.
863	*/
864	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
865	{
866	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
867	if (RT_SUCCESS(rc))
868	{ /* likely */ }
869	else
870	return rc;
871	}
872
873	/*
874	* Clear the VMCS we are switching to if it has not already been cleared.
875	* This will initialize the VMCS launch state to "clear" required for loading it.
876	*
877	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
878	*/
879	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
880	{
881	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
882	if (RT_SUCCESS(rc))
883	{ /* likely */ }
884	else
885	return rc;
886	}
887
888	/*
889	* Finally, load the VMCS we are switching to.
890	*/
891	return hmR0VmxLoadVmcs(pVmcsInfoTo);
892	}
893	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
894
895
896	/**
897	* Updates the VM's last error record.
898	*
899	* If there was a VMX instruction error, reads the error data from the VMCS and
900	* updates VCPU's last error record as well.
901	*
902	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
903	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
904	* VERR_VMX_INVALID_VMCS_FIELD.
905	* @param rc The error code.
906	*/
907	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
908	{
909	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
910	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
911	{
912	AssertPtrReturnVoid(pVCpu);
913	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
914	}
915	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
916	}
917
918
919	#ifdef VBOX_STRICT
920	/**
921	* Reads the VM-entry interruption-information field from the VMCS into the VMX
922	* transient structure.
923	*
924	* @returns VBox status code.
925	* @param pVmxTransient The VMX-transient structure.
926	*
927	* @remarks No-long-jump zone!!!
928	*/
929	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
930	{
931	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
932	AssertRCReturn(rc, rc);
933	return VINF_SUCCESS;
934	}
935
936
937	/**
938	* Reads the VM-entry exception error code field from the VMCS into
939	* the VMX transient structure.
940	*
941	* @returns VBox status code.
942	* @param pVmxTransient The VMX-transient structure.
943	*
944	* @remarks No-long-jump zone!!!
945	*/
946	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
947	{
948	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
949	AssertRCReturn(rc, rc);
950	return VINF_SUCCESS;
951	}
952
953
954	/**
955	* Reads the VM-entry exception error code field from the VMCS into
956	* the VMX transient structure.
957	*
958	* @returns VBox status code.
959	* @param pVmxTransient The VMX-transient structure.
960	*
961	* @remarks No-long-jump zone!!!
962	*/
963	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
964	{
965	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
966	AssertRCReturn(rc, rc);
967	return VINF_SUCCESS;
968	}
969	#endif /* VBOX_STRICT */
970
971
972	/**
973	* Reads the VM-exit interruption-information field from the VMCS into the VMX
974	* transient structure.
975	*
976	* @returns VBox status code.
977	* @param pVmxTransient The VMX-transient structure.
978	*/
979	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
980	{
981	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
982	{
983	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
984	AssertRCReturn(rc,rc);
985	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
986	}
987	return VINF_SUCCESS;
988	}
989
990
991	/**
992	* Reads the VM-exit interruption error code from the VMCS into the VMX
993	* transient structure.
994	*
995	* @returns VBox status code.
996	* @param pVmxTransient The VMX-transient structure.
997	*/
998	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
999	{
1000	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1001	{
1002	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1003	AssertRCReturn(rc, rc);
1004	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1005	}
1006	return VINF_SUCCESS;
1007	}
1008
1009
1010	/**
1011	* Reads the VM-exit instruction length field from the VMCS into the VMX
1012	* transient structure.
1013	*
1014	* @returns VBox status code.
1015	* @param pVmxTransient The VMX-transient structure.
1016	*/
1017	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1018	{
1019	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1020	{
1021	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1022	AssertRCReturn(rc, rc);
1023	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1024	}
1025	return VINF_SUCCESS;
1026	}
1027
1028
1029	/**
1030	* Reads the VM-exit instruction-information field from the VMCS into
1031	* the VMX transient structure.
1032	*
1033	* @returns VBox status code.
1034	* @param pVmxTransient The VMX-transient structure.
1035	*/
1036	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1037	{
1038	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1039	{
1040	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1041	AssertRCReturn(rc, rc);
1042	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1043	}
1044	return VINF_SUCCESS;
1045	}
1046
1047
1048	/**
1049	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1050	*
1051	* @returns VBox status code.
1052	* @param pVCpu The cross context virtual CPU structure of the
1053	* calling EMT. (Required for the VMCS cache case.)
1054	* @param pVmxTransient The VMX-transient structure.
1055	*/
1056	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1057	{
1058	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1059	{
1060	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1061	AssertRCReturn(rc, rc);
1062	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1063	}
1064	return VINF_SUCCESS;
1065	}
1066
1067
1068	/**
1069	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1070	*
1071	* @returns VBox status code.
1072	* @param pVCpu The cross context virtual CPU structure of the
1073	* calling EMT. (Required for the VMCS cache case.)
1074	* @param pVmxTransient The VMX-transient structure.
1075	*/
1076	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1077	{
1078	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1079	{
1080	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1081	AssertRCReturn(rc, rc);
1082	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1083	}
1084	return VINF_SUCCESS;
1085	}
1086
1087
1088	/**
1089	* Reads the IDT-vectoring information field from the VMCS into the VMX
1090	* transient structure.
1091	*
1092	* @returns VBox status code.
1093	* @param pVmxTransient The VMX-transient structure.
1094	*
1095	* @remarks No-long-jump zone!!!
1096	*/
1097	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1098	{
1099	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1100	{
1101	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1102	AssertRCReturn(rc, rc);
1103	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1104	}
1105	return VINF_SUCCESS;
1106	}
1107
1108
1109	/**
1110	* Reads the IDT-vectoring error code from the VMCS into the VMX
1111	* transient structure.
1112	*
1113	* @returns VBox status code.
1114	* @param pVmxTransient The VMX-transient structure.
1115	*/
1116	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1117	{
1118	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1119	{
1120	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1121	AssertRCReturn(rc, rc);
1122	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1123	}
1124	return VINF_SUCCESS;
1125	}
1126
1127
1128	/**
1129	* Enters VMX root mode operation on the current CPU.
1130	*
1131	* @returns VBox status code.
1132	* @param pVM The cross context VM structure. Can be
1133	* NULL, after a resume.
1134	* @param HCPhysCpuPage Physical address of the VMXON region.
1135	* @param pvCpuPage Pointer to the VMXON region.
1136	*/
1137	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1138	{
1139	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1140	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1141	Assert(pvCpuPage);
1142	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1143
1144	if (pVM)
1145	{
1146	/* Write the VMCS revision identifier to the VMXON region. */
1147	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1148	}
1149
1150	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1151	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1152
1153	/* Enable the VMX bit in CR4 if necessary. */
1154	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1155
1156	/* Enter VMX root mode. */
1157	int rc = VMXEnable(HCPhysCpuPage);
1158	if (RT_FAILURE(rc))
1159	{
1160	if (!(uOldCr4 & X86_CR4_VMXE))
1161	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1162
1163	if (pVM)
1164	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1165	}
1166
1167	/* Restore interrupts. */
1168	ASMSetFlags(fEFlags);
1169	return rc;
1170	}
1171
1172
1173	/**
1174	* Exits VMX root mode operation on the current CPU.
1175	*
1176	* @returns VBox status code.
1177	*/
1178	static int hmR0VmxLeaveRootMode(void)
1179	{
1180	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1181
1182	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1183	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1184
1185	/* If we're for some reason not in VMX root mode, then don't leave it. */
1186	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1187
1188	int rc;
1189	if (uHostCR4 & X86_CR4_VMXE)
1190	{
1191	/* Exit VMX root mode and clear the VMX bit in CR4. */
1192	VMXDisable();
1193	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1194	rc = VINF_SUCCESS;
1195	}
1196	else
1197	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1198
1199	/* Restore interrupts. */
1200	ASMSetFlags(fEFlags);
1201	return rc;
1202	}
1203
1204
1205	/**
1206	* Allocates and maps a physically contiguous page. The allocated page is
1207	* zero'd out (used by various VT-x structures).
1208	*
1209	* @returns IPRT status code.
1210	* @param pMemObj Pointer to the ring-0 memory object.
1211	* @param ppVirt Where to store the virtual address of the
1212	* allocation.
1213	* @param pHCPhys Where to store the physical address of the
1214	* allocation.
1215	*/
1216	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1217	{
1218	AssertPtr(pMemObj);
1219	AssertPtr(ppVirt);
1220	AssertPtr(pHCPhys);
1221	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1222	if (RT_FAILURE(rc))
1223	return rc;
1224	ppVirt = RTR0MemObjAddress(pMemObj);
1225	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1226	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1227	return VINF_SUCCESS;
1228	}
1229
1230
1231	/**
1232	* Frees and unmaps an allocated, physical page.
1233	*
1234	* @param pMemObj Pointer to the ring-0 memory object.
1235	* @param ppVirt Where to re-initialize the virtual address of
1236	* allocation as 0.
1237	* @param pHCPhys Where to re-initialize the physical address of the
1238	* allocation as 0.
1239	*/
1240	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1241	{
1242	AssertPtr(pMemObj);
1243	AssertPtr(ppVirt);
1244	AssertPtr(pHCPhys);
1245	/* NULL is valid, accepted and ignored by the free function below. */
1246	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1247	*pMemObj = NIL_RTR0MEMOBJ;
1248	*ppVirt = NULL;
1249	*pHCPhys = NIL_RTHCPHYS;
1250	}
1251
1252
1253	/**
1254	* Initializes a VMCS info. object.
1255	*
1256	* @param pVmcsInfo The VMCS info. object.
1257	*/
1258	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1259	{
1260	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1261
1262	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1265	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1266	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1267	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1272	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1273	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1274	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1275	}
1276
1277
1278	/**
1279	* Frees the VT-x structures for a VMCS info. object.
1280	*
1281	* @param pVM The cross context VM structure.
1282	* @param pVmcsInfo The VMCS info. object.
1283	*/
1284	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1285	{
1286	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1287
1288	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1290
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1292	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1293	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1294
1295	hmR0VmxInitVmcsInfo(pVmcsInfo);
1296	}
1297
1298
1299	/**
1300	* Allocates the VT-x structures for a VMCS info. object.
1301	*
1302	* @returns VBox status code.
1303	* @param pVCpu The cross context virtual CPU structure.
1304	* @param pVmcsInfo The VMCS info. object.
1305	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1306	*/
1307	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1308	{
1309	PVM pVM = pVCpu->CTX_SUFF(pVM);
1310
1311	/* Allocate the guest VM control structure (VMCS). */
1312	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1313	if (RT_SUCCESS(rc))
1314	{
1315	if (!fIsNstGstVmcs)
1316	{
1317	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1318	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1319	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1320	{
1321	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1322	NULL /* pR3Ptr /, NULL / pRCPtr */);
1323	}
1324	}
1325	else
1326	{
1327	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1328	Assert(!pVmcsInfo->pbVirtApic);
1329	}
1330
1331	if (RT_SUCCESS(rc))
1332	{
1333	/*
1334	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1335	* transparent accesses of specific MSRs.
1336	*
1337	* If the condition for enabling MSR bitmaps changes here, don't forget to
1338	* update HMIsMsrBitmapActive().
1339	*
1340	* We don't share MSR bitmaps between the guest and nested-guest as we then
1341	* don't need to care about carefully restoring the guest MSR bitmap.
1342	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1343	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1344	*/
1345	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1346	{
1347	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1348	if (RT_SUCCESS(rc))
1349	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1350	}
1351
1352	if (RT_SUCCESS(rc))
1353	{
1354	/*
1355	* Allocate the VM-entry MSR-load area for the guest MSRs.
1356	*
1357	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1358	* the guest and nested-guest.
1359	*/
1360	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1361	&pVmcsInfo->HCPhysGuestMsrLoad);
1362	if (RT_SUCCESS(rc))
1363	{
1364	/*
1365	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1366	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1367	*/
1368	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1369	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1370	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1371
1372	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1373	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1374	&pVmcsInfo->HCPhysHostMsrLoad);
1375	}
1376	}
1377	}
1378	}
1379
1380	return rc;
1381	}
1382
1383
1384	/**
1385	* Free all VT-x structures for the VM.
1386	*
1387	* @returns IPRT status code.
1388	* @param pVM The cross context VM structure.
1389	*/
1390	static void hmR0VmxStructsFree(PVM pVM)
1391	{
1392	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1394	#endif
1395	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1396
1397	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1398	{
1399	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1400	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1401	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1402	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1403	if (pVM->cpum.ro.GuestFeatures.fVmx)
1404	{
1405	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1406	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1407	}
1408	#endif
1409	}
1410	}
1411
1412
1413	/**
1414	* Allocate all VT-x structures for the VM.
1415	*
1416	* @returns IPRT status code.
1417	* @param pVM The cross context VM structure.
1418	*/
1419	static int hmR0VmxStructsAlloc(PVM pVM)
1420	{
1421	/*
1422	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1423	* The VMCS size cannot be more than 4096 bytes.
1424	*
1425	* See Intel spec. Appendix A.1 "Basic VMX Information".
1426	*/
1427	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1428	if (cbVmcs <= X86_PAGE_4K_SIZE)
1429	{ /* likely */ }
1430	else
1431	{
1432	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1433	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1434	}
1435
1436	/*
1437	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1438	*/
1439	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1440	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1441	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1442	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1443	#endif
1444
1445	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1446	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1447	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1448
1449	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1450	{
1451	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1452	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1453	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1454	}
1455
1456	/*
1457	* Allocate per-VM VT-x structures.
1458	*/
1459	int rc = VINF_SUCCESS;
1460	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1461	/* Allocate crash-dump magic scratch page. */
1462	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1463	if (RT_FAILURE(rc))
1464	{
1465	hmR0VmxStructsFree(pVM);
1466	return rc;
1467	}
1468	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1469	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1470	#endif
1471
1472	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1473	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1474	{
1475	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1476	&pVM->hm.s.vmx.HCPhysApicAccess);
1477	if (RT_FAILURE(rc))
1478	{
1479	hmR0VmxStructsFree(pVM);
1480	return rc;
1481	}
1482	}
1483
1484	/*
1485	* Initialize per-VCPU VT-x structures.
1486	*/
1487	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1488	{
1489	/* Allocate the guest VMCS structures. */
1490	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1491	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1492	if (RT_SUCCESS(rc))
1493	{
1494	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1495	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1496	if (pVM->cpum.ro.GuestFeatures.fVmx)
1497	{
1498	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1499	if (RT_SUCCESS(rc))
1500	{ /* likely */ }
1501	else
1502	break;
1503	}
1504	#endif
1505	}
1506	else
1507	break;
1508	}
1509
1510	if (RT_FAILURE(rc))
1511	{
1512	hmR0VmxStructsFree(pVM);
1513	return rc;
1514	}
1515
1516	return VINF_SUCCESS;
1517	}
1518
1519
1520	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1521	/**
1522	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1523	*
1524	* @returns @c true if the MSR is intercepted, @c false otherwise.
1525	* @param pvMsrBitmap The MSR bitmap.
1526	* @param offMsr The MSR byte offset.
1527	* @param iBit The bit offset from the byte offset.
1528	*/
1529	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1530	{
1531	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1532	Assert(pbMsrBitmap);
1533	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1534	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1535	}
1536	#endif
1537
1538
1539	/**
1540	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1541	*
1542	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1543	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1544	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1545	* the read/write access of this MSR.
1546	*
1547	* @param pVCpu The cross context virtual CPU structure.
1548	* @param pVmcsInfo The VMCS info. object.
1549	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1550	* @param idMsr The MSR value.
1551	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1552	* include both a read -and- a write permission!
1553	*
1554	* @sa CPUMGetVmxMsrPermission.
1555	* @remarks Can be called with interrupts disabled.
1556	*/
1557	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1558	{
1559	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1560	Assert(pbMsrBitmap);
1561	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1562
1563	/*
1564	* MSR-bitmap Layout:
1565	* Byte index MSR range Interpreted as
1566	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1567	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1568	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1569	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1570	*
1571	* A bit corresponding to an MSR within the above range causes a VM-exit
1572	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1573	* the MSR range, it always cause a VM-exit.
1574	*
1575	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1576	*/
1577	uint16_t const offBitmapRead = 0;
1578	uint16_t const offBitmapWrite = 0x800;
1579	uint16_t offMsr;
1580	int32_t iBit;
1581	if (idMsr <= UINT32_C(0x00001fff))
1582	{
1583	offMsr = 0;
1584	iBit = idMsr;
1585	}
1586	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1587	{
1588	offMsr = 0x400;
1589	iBit = idMsr - UINT32_C(0xc0000000);
1590	}
1591	else
1592	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1593
1594	/*
1595	* Set the MSR read permission.
1596	*/
1597	uint16_t const offMsrRead = offBitmapRead + offMsr;
1598	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1599	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1600	{
1601	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1602	bool const fClear = !fIsNstGstVmcs ? true
1603	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1604	#else
1605	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1606	bool const fClear = true;
1607	#endif
1608	if (fClear)
1609	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1610	}
1611	else
1612	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1613
1614	/*
1615	* Set the MSR write permission.
1616	*/
1617	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1618	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1619	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1620	{
1621	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1622	bool const fClear = !fIsNstGstVmcs ? true
1623	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1624	#else
1625	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1626	bool const fClear = true;
1627	#endif
1628	if (fClear)
1629	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631	else
1632	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1633	}
1634
1635
1636	/**
1637	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1638	* area.
1639	*
1640	* @returns VBox status code.
1641	* @param pVCpu The cross context virtual CPU structure.
1642	* @param pVmcsInfo The VMCS info. object.
1643	* @param cMsrs The number of MSRs.
1644	*/
1645	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1646	{
1647	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1648	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1649	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1650	{
1651	/* Commit the MSR counts to the VMCS and update the cache. */
1652	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1653	{
1654	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1655	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1657	AssertRCReturn(rc, rc);
1658
1659	pVmcsInfo->cEntryMsrLoad = cMsrs;
1660	pVmcsInfo->cExitMsrStore = cMsrs;
1661	pVmcsInfo->cExitMsrLoad = cMsrs;
1662	}
1663	return VINF_SUCCESS;
1664	}
1665
1666	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1667	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1668	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1669	}
1670
1671
1672	/**
1673	* Adds a new (or updates the value of an existing) guest/host MSR
1674	* pair to be swapped during the world-switch as part of the
1675	* auto-load/store MSR area in the VMCS.
1676	*
1677	* @returns VBox status code.
1678	* @param pVCpu The cross context virtual CPU structure.
1679	* @param pVmxTransient The VMX-transient structure.
1680	* @param idMsr The MSR.
1681	* @param uGuestMsrValue Value of the guest MSR.
1682	* @param fSetReadWrite Whether to set the guest read/write access of this
1683	* MSR (thus not causing a VM-exit).
1684	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1685	* necessary.
1686	*/
1687	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1688	bool fSetReadWrite, bool fUpdateHostMsr)
1689	{
1690	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1691	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1692	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1693	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1694	uint32_t i;
1695
1696	/* Paranoia. */
1697	Assert(pGuestMsrLoad);
1698
1699	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1700
1701	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1702	for (i = 0; i < cMsrs; i++)
1703	{
1704	if (pGuestMsrLoad[i].u32Msr == idMsr)
1705	break;
1706	}
1707
1708	bool fAdded = false;
1709	if (i == cMsrs)
1710	{
1711	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1712	++cMsrs;
1713	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1714	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1715
1716	/* Set the guest to read/write this MSR without causing VM-exits. */
1717	if ( fSetReadWrite
1718	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1719	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1720
1721	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1722	fAdded = true;
1723	}
1724
1725	/* Update the MSR value for the newly added or already existing MSR. */
1726	pGuestMsrLoad[i].u32Msr = idMsr;
1727	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1728
1729	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1730	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1731	{
1732	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1733	pGuestMsrStore[i].u32Msr = idMsr;
1734	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1735	}
1736
1737	/* Update the corresponding slot in the host MSR area. */
1738	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1739	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1740	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1741	pHostMsr[i].u32Msr = idMsr;
1742
1743	/*
1744	* Only if the caller requests to update the host MSR value AND we've newly added the
1745	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1746	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1747	*
1748	* We do this for performance reasons since reading MSRs may be quite expensive.
1749	*/
1750	if (fAdded)
1751	{
1752	if (fUpdateHostMsr)
1753	{
1754	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1755	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1756	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1757	}
1758	else
1759	{
1760	/* Someone else can do the work. */
1761	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1762	}
1763	}
1764	return VINF_SUCCESS;
1765	}
1766
1767
1768	/**
1769	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1770	* auto-load/store MSR area in the VMCS.
1771	*
1772	* @returns VBox status code.
1773	* @param pVCpu The cross context virtual CPU structure.
1774	* @param pVmxTransient The VMX-transient structure.
1775	* @param idMsr The MSR.
1776	*/
1777	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1778	{
1779	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1780	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1781	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1782	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1783
1784	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1785
1786	for (uint32_t i = 0; i < cMsrs; i++)
1787	{
1788	/* Find the MSR. */
1789	if (pGuestMsrLoad[i].u32Msr == idMsr)
1790	{
1791	/*
1792	* If it's the last MSR, we only need to reduce the MSR count.
1793	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1794	*/
1795	if (i < cMsrs - 1)
1796	{
1797	/* Remove it from the VM-entry MSR-load area. */
1798	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1799	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1800
1801	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1802	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1803	{
1804	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1805	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1806	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1807	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1808	}
1809
1810	/* Remove it from the VM-exit MSR-load area. */
1811	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1812	Assert(pHostMsr[i].u32Msr == idMsr);
1813	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1814	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1815	}
1816
1817	/* Reduce the count to reflect the removed MSR and bail. */
1818	--cMsrs;
1819	break;
1820	}
1821	}
1822
1823	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
1824	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1825	{
1826	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1827	AssertRCReturn(rc, rc);
1828
1829	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1830	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1831	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1832
1833	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1834	return VINF_SUCCESS;
1835	}
1836
1837	return VERR_NOT_FOUND;
1838	}
1839
1840
1841	/**
1842	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1843	*
1844	* @returns @c true if found, @c false otherwise.
1845	* @param pVmcsInfo The VMCS info. object.
1846	* @param idMsr The MSR to find.
1847	*/
1848	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1849	{
1850	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1851	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1852	Assert(pMsrs);
1853	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
1854	for (uint32_t i = 0; i < cMsrs; i++)
1855	{
1856	if (pMsrs[i].u32Msr == idMsr)
1857	return true;
1858	}
1859	return false;
1860	}
1861
1862
1863	/**
1864	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1865	*
1866	* @param pVCpu The cross context virtual CPU structure.
1867	* @param pVmcsInfo The VMCS info. object.
1868	*
1869	* @remarks No-long-jump zone!!!
1870	*/
1871	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1872	{
1873	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1874
1875	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1876	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1877	Assert(pHostMsrLoad);
1878	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
1879	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
1880	for (uint32_t i = 0; i < cMsrs; i++)
1881	{
1882	/*
1883	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1884	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1885	*/
1886	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
1887	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1888	else
1889	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
1890	}
1891	}
1892
1893
1894	/**
1895	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1896	* perform lazy restoration of the host MSRs while leaving VT-x.
1897	*
1898	* @param pVCpu The cross context virtual CPU structure.
1899	*
1900	* @remarks No-long-jump zone!!!
1901	*/
1902	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1903	{
1904	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1905
1906	/*
1907	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1908	*/
1909	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1910	{
1911	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1912	#if HC_ARCH_BITS == 64
1913	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1914	{
1915	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1916	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1917	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1918	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1919	}
1920	#endif
1921	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1922	}
1923	}
1924
1925
1926	/**
1927	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1928	* lazily while leaving VT-x.
1929	*
1930	* @returns true if it does, false otherwise.
1931	* @param pVCpu The cross context virtual CPU structure.
1932	* @param idMsr The MSR to check.
1933	*/
1934	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
1935	{
1936	NOREF(pVCpu);
1937	#if HC_ARCH_BITS == 64
1938	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1939	{
1940	switch (idMsr)
1941	{
1942	case MSR_K8_LSTAR:
1943	case MSR_K6_STAR:
1944	case MSR_K8_SF_MASK:
1945	case MSR_K8_KERNEL_GS_BASE:
1946	return true;
1947	}
1948	}
1949	#else
1950	RT_NOREF(pVCpu, idMsr);
1951	#endif
1952	return false;
1953	}
1954
1955
1956	/**
1957	* Loads a set of guests MSRs to allow read/passthru to the guest.
1958	*
1959	* The name of this function is slightly confusing. This function does NOT
1960	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1961	* common prefix for functions dealing with "lazy restoration" of the shared
1962	* MSRs.
1963	*
1964	* @param pVCpu The cross context virtual CPU structure.
1965	*
1966	* @remarks No-long-jump zone!!!
1967	*/
1968	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1969	{
1970	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1971	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1972
1973	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1974	#if HC_ARCH_BITS == 64
1975	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1976	{
1977	/*
1978	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1979	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1980	* we can skip a few MSR writes.
1981	*
1982	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1983	* guest MSR values in the guest-CPU context might be different to what's currently
1984	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1985	* CPU, see @bugref{8728}.
1986	*/
1987	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1988	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1989	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1990	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1991	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1992	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1993	{
1994	#ifdef VBOX_STRICT
1995	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1996	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1997	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1998	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1999	#endif
2000	}
2001	else
2002	{
2003	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2004	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2005	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2006	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2007	}
2008	}
2009	#endif
2010	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2011	}
2012
2013
2014	/**
2015	* Performs lazy restoration of the set of host MSRs if they were previously
2016	* loaded with guest MSR values.
2017	*
2018	* @param pVCpu The cross context virtual CPU structure.
2019	*
2020	* @remarks No-long-jump zone!!!
2021	* @remarks The guest MSRs should have been saved back into the guest-CPU
2022	* context by hmR0VmxImportGuestState()!!!
2023	*/
2024	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2025	{
2026	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2027	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2028
2029	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2030	{
2031	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2032	#if HC_ARCH_BITS == 64
2033	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2034	{
2035	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2036	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2037	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2038	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2039	}
2040	#endif
2041	}
2042	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2043	}
2044
2045
2046	/**
2047	* Verifies that our cached values of the VMCS fields are all consistent with
2048	* what's actually present in the VMCS.
2049	*
2050	* @returns VBox status code.
2051	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2052	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2053	* VMCS content. HMCPU error-field is
2054	* updated, see VMX_VCI_XXX.
2055	* @param pVCpu The cross context virtual CPU structure.
2056	* @param pVmcsInfo The VMCS info. object.
2057	*/
2058	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2059	{
2060	uint32_t u32Val;
2061	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2062	AssertRCReturn(rc, rc);
2063	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2064	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2065	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2066	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2067
2068	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2069	AssertRCReturn(rc, rc);
2070	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2071	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2072	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2073	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2074
2075	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2076	AssertRCReturn(rc, rc);
2077	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2078	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2079	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2080	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2081
2082	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2083	AssertRCReturn(rc, rc);
2084	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2085	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2086	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2087	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2088
2089	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2090	{
2091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2092	AssertRCReturn(rc, rc);
2093	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2094	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2095	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2096	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2097	}
2098
2099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2100	AssertRCReturn(rc, rc);
2101	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2102	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2103	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2104	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2105
2106	uint64_t u64Val;
2107	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2108	AssertRCReturn(rc, rc);
2109	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2110	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2111	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2112	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2113
2114	return VINF_SUCCESS;
2115	}
2116
2117
2118	#ifdef VBOX_STRICT
2119	/**
2120	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2121	*
2122	* @param pVCpu The cross context virtual CPU structure.
2123	* @param pVmcsInfo The VMCS info. object.
2124	*/
2125	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2126	{
2127	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2128
2129	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2130	{
2131	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2132	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2133	uint64_t uVmcsEferMsrVmcs;
2134	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2135	AssertRC(rc);
2136
2137	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2138	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2139	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2140	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2141	}
2142	}
2143
2144
2145	/**
2146	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2147	* VMCS are correct.
2148	*
2149	* @param pVCpu The cross context virtual CPU structure.
2150	* @param pVmcsInfo The VMCS info. object.
2151	*/
2152	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2153	{
2154	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2155
2156	/* Read the various MSR-area counts from the VMCS. */
2157	uint32_t cEntryLoadMsrs;
2158	uint32_t cExitStoreMsrs;
2159	uint32_t cExitLoadMsrs;
2160	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2162	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2163
2164	/* Verify all the MSR counts are the same. */
2165	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2166	Assert(cExitStoreMsrs == cExitLoadMsrs);
2167	uint32_t const cMsrs = cExitLoadMsrs;
2168
2169	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2170	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2171
2172	/* Verify the MSR counts are within the allocated page size. */
2173	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2174
2175	/* Verify the relevant contents of the MSR areas match. */
2176	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2177	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2178	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2179	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2180	for (uint32_t i = 0; i < cMsrs; i++)
2181	{
2182	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2183	if (fSeparateExitMsrStorePage)
2184	{
2185	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2186	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2187	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2188	}
2189
2190	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2191	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2192	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2193
2194	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2195	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2196	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2197	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2198
2199	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2200	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2201	if (fIsEferMsr)
2202	{
2203	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2204	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2205	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2206	}
2207
2208	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2209	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2210	{
2211	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2212	if (fIsEferMsr)
2213	{
2214	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2215	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2216	}
2217	else
2218	{
2219	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2220	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2221	}
2222	}
2223
2224	/* Move to the next MSR. */
2225	pHostMsrLoad++;
2226	pGuestMsrLoad++;
2227	pGuestMsrStore++;
2228	}
2229	}
2230	#endif /* VBOX_STRICT */
2231
2232
2233	/**
2234	* Flushes the TLB using EPT.
2235	*
2236	* @returns VBox status code.
2237	* @param pVCpu The cross context virtual CPU structure of the calling
2238	* EMT. Can be NULL depending on @a enmTlbFlush.
2239	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2240	* enmTlbFlush.
2241	* @param enmTlbFlush Type of flush.
2242	*
2243	* @remarks Caller is responsible for making sure this function is called only
2244	* when NestedPaging is supported and providing @a enmTlbFlush that is
2245	* supported by the CPU.
2246	* @remarks Can be called with interrupts disabled.
2247	*/
2248	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2249	{
2250	uint64_t au64Descriptor[2];
2251	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2252	au64Descriptor[0] = 0;
2253	else
2254	{
2255	Assert(pVCpu);
2256	Assert(pVmcsInfo);
2257	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2258	}
2259	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2260
2261	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2262	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2263
2264	if ( RT_SUCCESS(rc)
2265	&& pVCpu)
2266	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2267	}
2268
2269
2270	/**
2271	* Flushes the TLB using VPID.
2272	*
2273	* @returns VBox status code.
2274	* @param pVCpu The cross context virtual CPU structure of the calling
2275	* EMT. Can be NULL depending on @a enmTlbFlush.
2276	* @param enmTlbFlush Type of flush.
2277	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2278	* on @a enmTlbFlush).
2279	*
2280	* @remarks Can be called with interrupts disabled.
2281	*/
2282	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2283	{
2284	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2285
2286	uint64_t au64Descriptor[2];
2287	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2288	{
2289	au64Descriptor[0] = 0;
2290	au64Descriptor[1] = 0;
2291	}
2292	else
2293	{
2294	AssertPtr(pVCpu);
2295	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2296	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2297	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2298	au64Descriptor[1] = GCPtr;
2299	}
2300
2301	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2302	AssertMsg(rc == VINF_SUCCESS,
2303	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2304
2305	if ( RT_SUCCESS(rc)
2306	&& pVCpu)
2307	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2308	NOREF(rc);
2309	}
2310
2311
2312	/**
2313	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2314	* otherwise there is nothing really to invalidate.
2315	*
2316	* @returns VBox status code.
2317	* @param pVCpu The cross context virtual CPU structure.
2318	* @param GCVirt Guest virtual address of the page to invalidate.
2319	*/
2320	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2321	{
2322	AssertPtr(pVCpu);
2323	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2324
2325	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2326	{
2327	/*
2328	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2329	* the EPT case. See @bugref{6043} and @bugref{6177}.
2330	*
2331	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2332	* as this function maybe called in a loop with individual addresses.
2333	*/
2334	PVM pVM = pVCpu->CTX_SUFF(pVM);
2335	if (pVM->hm.s.vmx.fVpid)
2336	{
2337	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2338
2339	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2340	/*
2341	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2342	* where executing INVVPID outside 64-bit mode does not flush translations of
2343	* 64-bit linear addresses, see @bugref{6208#c72}.
2344	*/
2345	if (RT_HI_U32(GCVirt))
2346	fVpidFlush = false;
2347	#endif
2348
2349	if (fVpidFlush)
2350	{
2351	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2352	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2353	}
2354	else
2355	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2356	}
2357	else if (pVM->hm.s.fNestedPaging)
2358	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2359	}
2360
2361	return VINF_SUCCESS;
2362	}
2363
2364
2365	/**
2366	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2367	* case where neither EPT nor VPID is supported by the CPU.
2368	*
2369	* @param pHostCpu The HM physical-CPU structure.
2370	* @param pVCpu The cross context virtual CPU structure.
2371	*
2372	* @remarks Called with interrupts disabled.
2373	*/
2374	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2375	{
2376	AssertPtr(pVCpu);
2377	AssertPtr(pHostCpu);
2378
2379	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2380
2381	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2382	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2383	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2384	pVCpu->hm.s.fForceTLBFlush = false;
2385	return;
2386	}
2387
2388
2389	/**
2390	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2391	*
2392	* @param pHostCpu The HM physical-CPU structure.
2393	* @param pVCpu The cross context virtual CPU structure.
2394	* @param pVmcsInfo The VMCS info. object.
2395	*
2396	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2397	* nomenclature. The reason is, to avoid confusion in compare statements
2398	* since the host-CPU copies are named "ASID".
2399	*
2400	* @remarks Called with interrupts disabled.
2401	*/
2402	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2403	{
2404	#ifdef VBOX_WITH_STATISTICS
2405	bool fTlbFlushed = false;
2406	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2407	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2408	if (!fTlbFlushed) \
2409	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2410	} while (0)
2411	#else
2412	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2413	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2414	#endif
2415
2416	AssertPtr(pVCpu);
2417	AssertPtr(pHostCpu);
2418	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2419
2420	PVM pVM = pVCpu->CTX_SUFF(pVM);
2421	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2422	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2423	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2424
2425	/*
2426	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2427	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2428	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2429	* cannot reuse the current ASID anymore.
2430	*/
2431	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2432	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2433	{
2434	++pHostCpu->uCurrentAsid;
2435	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2436	{
2437	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2438	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2439	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2440	}
2441
2442	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2443	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2444	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2445
2446	/*
2447	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2448	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2449	*/
2450	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2451	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2452	HMVMX_SET_TAGGED_TLB_FLUSHED();
2453	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2454	}
2455	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2456	{
2457	/*
2458	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2459	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2460	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2461	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2462	* mappings, see @bugref{6568}.
2463	*
2464	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2465	*/
2466	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2467	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2468	HMVMX_SET_TAGGED_TLB_FLUSHED();
2469	}
2470
2471	pVCpu->hm.s.fForceTLBFlush = false;
2472	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2473
2474	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2475	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2476	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2477	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2478	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2479	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2480	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2481	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2482	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2483
2484	/* Update VMCS with the VPID. */
2485	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2486	AssertRC(rc);
2487
2488	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2489	}
2490
2491
2492	/**
2493	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2494	*
2495	* @param pHostCpu The HM physical-CPU structure.
2496	* @param pVCpu The cross context virtual CPU structure.
2497	* @param pVmcsInfo The VMCS info. object.
2498	*
2499	* @remarks Called with interrupts disabled.
2500	*/
2501	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2502	{
2503	AssertPtr(pVCpu);
2504	AssertPtr(pHostCpu);
2505	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2506	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2507	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2508
2509	/*
2510	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2511	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2512	*/
2513	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2514	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2515	{
2516	pVCpu->hm.s.fForceTLBFlush = true;
2517	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2518	}
2519
2520	/* Check for explicit TLB flushes. */
2521	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2522	{
2523	pVCpu->hm.s.fForceTLBFlush = true;
2524	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2525	}
2526
2527	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2528	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2529
2530	if (pVCpu->hm.s.fForceTLBFlush)
2531	{
2532	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2533	pVCpu->hm.s.fForceTLBFlush = false;
2534	}
2535	}
2536
2537
2538	/**
2539	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2540	*
2541	* @param pHostCpu The HM physical-CPU structure.
2542	* @param pVCpu The cross context virtual CPU structure.
2543	*
2544	* @remarks Called with interrupts disabled.
2545	*/
2546	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2547	{
2548	AssertPtr(pVCpu);
2549	AssertPtr(pHostCpu);
2550	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2551	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2552	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2553
2554	/*
2555	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2556	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2557	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2558	* cannot reuse the current ASID anymore.
2559	*/
2560	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2561	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2562	{
2563	pVCpu->hm.s.fForceTLBFlush = true;
2564	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2565	}
2566
2567	/* Check for explicit TLB flushes. */
2568	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2569	{
2570	/*
2571	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2572	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2573	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2574	* include fExplicitFlush's too) - an obscure corner case.
2575	*/
2576	pVCpu->hm.s.fForceTLBFlush = true;
2577	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2578	}
2579
2580	PVM pVM = pVCpu->CTX_SUFF(pVM);
2581	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2582	if (pVCpu->hm.s.fForceTLBFlush)
2583	{
2584	++pHostCpu->uCurrentAsid;
2585	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2586	{
2587	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2588	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2589	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2590	}
2591
2592	pVCpu->hm.s.fForceTLBFlush = false;
2593	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2594	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2595	if (pHostCpu->fFlushAsidBeforeUse)
2596	{
2597	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2598	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2599	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2600	{
2601	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2602	pHostCpu->fFlushAsidBeforeUse = false;
2603	}
2604	else
2605	{
2606	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2607	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2608	}
2609	}
2610	}
2611
2612	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2613	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2614	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2615	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2616	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2617	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2618	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2619
2620	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2621	AssertRC(rc);
2622	}
2623
2624
2625	/**
2626	* Flushes the guest TLB entry based on CPU capabilities.
2627	*
2628	* @param pHostCpu The HM physical-CPU structure.
2629	* @param pVCpu The cross context virtual CPU structure.
2630	* @param pVmcsInfo The VMCS info. object.
2631	*
2632	* @remarks Called with interrupts disabled.
2633	*/
2634	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2635	{
2636	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2637	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2638	#endif
2639	PVM pVM = pVCpu->CTX_SUFF(pVM);
2640	switch (pVM->hm.s.vmx.enmTlbFlushType)
2641	{
2642	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2643	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2644	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2645	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2646	default:
2647	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2648	break;
2649	}
2650	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2651	}
2652
2653
2654	/**
2655	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2656	* TLB entries from the host TLB before VM-entry.
2657	*
2658	* @returns VBox status code.
2659	* @param pVM The cross context VM structure.
2660	*/
2661	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2662	{
2663	/*
2664	* Determine optimal flush type for nested paging.
2665	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2666	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2667	*/
2668	if (pVM->hm.s.fNestedPaging)
2669	{
2670	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2671	{
2672	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2673	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2674	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2675	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2676	else
2677	{
2678	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683
2684	/* Make sure the write-back cacheable memory type for EPT is supported. */
2685	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2686	{
2687	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2688	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2690	}
2691
2692	/* EPT requires a page-walk length of 4. */
2693	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2694	{
2695	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2696	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2697	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2698	}
2699	}
2700	else
2701	{
2702	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2703	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2704	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2705	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2706	}
2707	}
2708
2709	/*
2710	* Determine optimal flush type for VPID.
2711	*/
2712	if (pVM->hm.s.vmx.fVpid)
2713	{
2714	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2715	{
2716	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2717	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2718	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2720	else
2721	{
2722	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2723	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2724	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2725	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2726	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2727	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2728	pVM->hm.s.vmx.fVpid = false;
2729	}
2730	}
2731	else
2732	{
2733	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2734	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2735	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2736	pVM->hm.s.vmx.fVpid = false;
2737	}
2738	}
2739
2740	/*
2741	* Setup the handler for flushing tagged-TLBs.
2742	*/
2743	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2744	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2745	else if (pVM->hm.s.fNestedPaging)
2746	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2747	else if (pVM->hm.s.vmx.fVpid)
2748	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2749	else
2750	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2751	return VINF_SUCCESS;
2752	}
2753
2754
2755	/**
2756	* Sets up the virtual-APIC page address for the VMCS.
2757	*
2758	* @returns VBox status code.
2759	* @param pVCpu The cross context virtual CPU structure.
2760	* @param pVmcsInfo The VMCS info. object.
2761	*/
2762	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2763	{
2764	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2765	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2766	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2767	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2768	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2769	}
2770
2771
2772	/**
2773	* Sets up the MSR-bitmap address for the VMCS.
2774	*
2775	* @returns VBox status code.
2776	* @param pVCpu The cross context virtual CPU structure.
2777	* @param pVmcsInfo The VMCS info. object.
2778	*/
2779	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2780	{
2781	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2782	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2783	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2784	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2785	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2786	}
2787
2788
2789	/**
2790	* Sets up the APIC-access page address for the VMCS.
2791	*
2792	* @returns VBox status code.
2793	* @param pVCpu The cross context virtual CPU structure.
2794	*/
2795	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2796	{
2797	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2798	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2799	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2800	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2801	}
2802
2803
2804	/**
2805	* Sets up the VMCS link pointer for the VMCS.
2806	*
2807	* @returns VBox status code.
2808	* @param pVCpu The cross context virtual CPU structure.
2809	* @param pVmcsInfo The VMCS info. object.
2810	*/
2811	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2812	{
2813	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2814	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2815	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2816	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2817	}
2818
2819
2820	/**
2821	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2822	* in the VMCS.
2823	*
2824	* @returns VBox status code.
2825	* @param pVCpu The cross context virtual CPU structure.
2826	* @param pVmcsInfo The VMCS info. object.
2827	*/
2828	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2829	{
2830	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2831
2832	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2833	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2834	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2835
2836	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2837	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2838	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2839
2840	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2841	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2842	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2843
2844	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2845	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2846	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2847	AssertRCReturn(rc, rc);
2848	return VINF_SUCCESS;
2849	}
2850
2851
2852	/**
2853	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2854	*
2855	* @param pVCpu The cross context virtual CPU structure.
2856	* @param pVmcsInfo The VMCS info. object.
2857	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2858	*/
2859	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2860	{
2861	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2862
2863	/*
2864	* The guest can access the following MSRs (read, write) without causing
2865	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2866	*/
2867	PVM pVM = pVCpu->CTX_SUFF(pVM);
2868	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
2869	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
2871	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2872	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
2873
2874	/*
2875	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2876	* associated with then. We never need to intercept access (writes need to be
2877	* executed without causing a VM-exit, reads will #GP fault anyway).
2878	*
2879	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2880	* read/write them. We swap the the guest/host MSR value using the
2881	* auto-load/store MSR area.
2882	*/
2883	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2884	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
2885	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2886	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
2887	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2888	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
2889
2890	#if HC_ARCH_BITS == 64
2891	/*
2892	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2893	* required for 64-bit guests.
2894	*/
2895	if (pVM->hm.s.fAllow64BitGuests)
2896	{
2897	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
2898	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
2899	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
2900	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
2901	}
2902	#endif
2903
2904	/*
2905	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2906	*/
2907	#ifdef VBOX_STRICT
2908	Assert(pVmcsInfo->pvMsrBitmap);
2909	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
2910	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
2911	#endif
2912	}
2913
2914
2915	/**
2916	* Sets up pin-based VM-execution controls in the VMCS.
2917	*
2918	* @returns VBox status code.
2919	* @param pVCpu The cross context virtual CPU structure.
2920	* @param pVmcsInfo The VMCS info. object.
2921	*/
2922	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2923	{
2924	PVM pVM = pVCpu->CTX_SUFF(pVM);
2925	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2926	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2927
2928	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2929	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2930
2931	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2932	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2933
2934	/* Enable the VMX-preemption timer. */
2935	if (pVM->hm.s.vmx.fUsePreemptTimer)
2936	{
2937	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2938	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2939	}
2940
2941	#if 0
2942	/* Enable posted-interrupt processing. */
2943	if (pVM->hm.s.fPostedIntrs)
2944	{
2945	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2946	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2947	fVal \|= VMX_PIN_CTL_POSTED_INT;
2948	}
2949	#endif
2950
2951	if ((fVal & fZap) != fVal)
2952	{
2953	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2954	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2955	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2956	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2957	}
2958
2959	/* Commit it to the VMCS and update our cache. */
2960	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2961	AssertRCReturn(rc, rc);
2962	pVmcsInfo->u32PinCtls = fVal;
2963
2964	return VINF_SUCCESS;
2965	}
2966
2967
2968	/**
2969	* Sets up secondary processor-based VM-execution controls in the VMCS.
2970	*
2971	* @returns VBox status code.
2972	* @param pVCpu The cross context virtual CPU structure.
2973	* @param pVmcsInfo The VMCS info. object.
2974	*/
2975	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2976	{
2977	PVM pVM = pVCpu->CTX_SUFF(pVM);
2978	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2979	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2980
2981	/* WBINVD causes a VM-exit. */
2982	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2983	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2984
2985	/* Enable EPT (aka nested-paging). */
2986	if (pVM->hm.s.fNestedPaging)
2987	fVal \|= VMX_PROC_CTLS2_EPT;
2988
2989	/* Enable the INVPCID instruction if supported by the hardware and we expose
2990	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2991	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2992	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2993	fVal \|= VMX_PROC_CTLS2_INVPCID;
2994
2995	/* Enable VPID. */
2996	if (pVM->hm.s.vmx.fVpid)
2997	fVal \|= VMX_PROC_CTLS2_VPID;
2998
2999	/* Enable unrestricted guest execution. */
3000	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3001	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3002
3003	#if 0
3004	if (pVM->hm.s.fVirtApicRegs)
3005	{
3006	/* Enable APIC-register virtualization. */
3007	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3008	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3009
3010	/* Enable virtual-interrupt delivery. */
3011	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3012	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3013	}
3014	#endif
3015
3016	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3017	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3018	* done dynamically. */
3019	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3020	{
3021	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3022	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3023	AssertRCReturn(rc, rc);
3024	}
3025
3026	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3027	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3028	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3029	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3030	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3031
3032	/* Enable Pause-Loop exiting. */
3033	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3034	&& pVM->hm.s.vmx.cPleGapTicks
3035	&& pVM->hm.s.vmx.cPleWindowTicks)
3036	{
3037	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3038
3039	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3040	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3041	AssertRCReturn(rc, rc);
3042	}
3043
3044	if ((fVal & fZap) != fVal)
3045	{
3046	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3047	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3048	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3049	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3050	}
3051
3052	/* Commit it to the VMCS and update our cache. */
3053	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3054	AssertRCReturn(rc, rc);
3055	pVmcsInfo->u32ProcCtls2 = fVal;
3056
3057	return VINF_SUCCESS;
3058	}
3059
3060
3061	/**
3062	* Sets up processor-based VM-execution controls in the VMCS.
3063	*
3064	* @returns VBox status code.
3065	* @param pVCpu The cross context virtual CPU structure.
3066	* @param pVmcsInfo The VMCS info. object.
3067	*/
3068	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3069	{
3070	PVM pVM = pVCpu->CTX_SUFF(pVM);
3071
3072	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3073	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3074
3075	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3076	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3077	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3078	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3079	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3080	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3081	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3082
3083	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3084	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3085	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3086	{
3087	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3088	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3089	}
3090
3091	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3092	if (!pVM->hm.s.fNestedPaging)
3093	{
3094	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3095	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3096	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3097	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3098	}
3099
3100	/* Use TPR shadowing if supported by the CPU. */
3101	if ( PDMHasApic(pVM)
3102	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3103	{
3104	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3105	/* CR8 writes cause a VM-exit based on TPR threshold. */
3106	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3107	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3108	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3109	AssertRCReturn(rc, rc);
3110	}
3111	else
3112	{
3113	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3114	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3115	if (pVM->hm.s.fAllow64BitGuests)
3116	{
3117	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3118	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3119	}
3120	}
3121
3122	/* Use MSR-bitmaps if supported by the CPU. */
3123	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3124	{
3125	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3126	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3127	AssertRCReturn(rc, rc);
3128	}
3129
3130	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3131	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3132	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3133
3134	if ((fVal & fZap) != fVal)
3135	{
3136	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3137	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3138	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3139	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3140	}
3141
3142	/* Commit it to the VMCS and update our cache. */
3143	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3144	AssertRCReturn(rc, rc);
3145	pVmcsInfo->u32ProcCtls = fVal;
3146
3147	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3148	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3149	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3150
3151	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3152	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3153	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3154
3155	/* Sanity check, should not really happen. */
3156	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3157	{ /* likely */ }
3158	else
3159	{
3160	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3161	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3162	}
3163
3164	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3165	return VINF_SUCCESS;
3166	}
3167
3168
3169	/**
3170	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3171	* Processor-based VM-execution) control fields in the VMCS.
3172	*
3173	* @returns VBox status code.
3174	* @param pVCpu The cross context virtual CPU structure.
3175	* @param pVmcsInfo The VMCS info. object.
3176	*/
3177	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3178	{
3179	/* Set the auto-load/store MSR area addresses in the VMCS. */
3180	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3181	if (RT_SUCCESS(rc))
3182	{
3183	/* Set the VMCS link pointer in the VMCS. */
3184	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3185	if (RT_SUCCESS(rc))
3186	{
3187	/* Set the CR0/CR4 guest/host mask. */
3188	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3189	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3190	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3191	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3192	if (RT_SUCCESS(rc))
3193	{
3194	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3195	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3196	return VINF_SUCCESS;
3197	}
3198	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3199	}
3200	else
3201	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3202	}
3203	else
3204	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3205	return rc;
3206	}
3207
3208
3209	/**
3210	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3211	*
3212	* We shall setup those exception intercepts that don't change during the
3213	* lifetime of the VM here. The rest are done dynamically while loading the
3214	* guest state.
3215	*
3216	* @returns VBox status code.
3217	* @param pVCpu The cross context virtual CPU structure.
3218	* @param pVmcsInfo The VMCS info. object.
3219	*/
3220	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3221	{
3222	/*
3223	* The following exceptions are always intercepted:
3224	*
3225	* #AC - To prevent the guest from hanging the CPU.
3226	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3227	* recursive #DBs can cause a CPU hang.
3228	* #PF - To sync our shadow page tables when nested-paging is not used.
3229	*/
3230	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3231	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3232	\| RT_BIT(X86_XCPT_DB)
3233	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3234
3235	/* Commit it to the VMCS. */
3236	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3237	AssertRCReturn(rc, rc);
3238
3239	/* Update our cache of the exception bitmap. */
3240	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3241	return VINF_SUCCESS;
3242	}
3243
3244
3245	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3246	/**
3247	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3248	*
3249	* @returns VBox status code.
3250	* @param pVCpu The cross context virtual CPU structure.
3251	* @param pVmcsInfo The VMCS info. object.
3252	*/
3253	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3254	{
3255	PVM pVM = pVCpu->CTX_SUFF(pVM);
3256	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3257	if (RT_SUCCESS(rc))
3258	{
3259	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3260	if (RT_SUCCESS(rc))
3261	{
3262	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3263	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3264	if (RT_SUCCESS(rc))
3265	{
3266	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3267	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3268	if (RT_SUCCESS(rc))
3269	return VINF_SUCCESS;
3270
3271	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3272	}
3273	else
3274	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3275	}
3276	else
3277	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3278	}
3279	else
3280	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3281
3282	return rc;
3283	}
3284	#endif
3285
3286
3287	/**
3288	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3289	* VMX.
3290	*
3291	* @returns VBox status code.
3292	* @param pVCpu The cross context virtual CPU structure.
3293	* @param pVmcsInfo The VMCS info. object.
3294	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3295	*/
3296	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3297	{
3298	Assert(pVmcsInfo);
3299	Assert(pVmcsInfo->pvVmcs);
3300	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3301
3302	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3303	PVM pVM = pVCpu->CTX_SUFF(pVM);
3304	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3305	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3306
3307	LogFlowFunc(("\n"));
3308
3309	/*
3310	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3311	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3312	*/
3313	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3314	if (RT_SUCCESS(rc))
3315	{
3316	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3317	if (RT_SUCCESS(rc))
3318	{
3319	if (!fIsNstGstVmcs)
3320	{
3321	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3322	if (RT_SUCCESS(rc))
3323	{
3324	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3325	if (RT_SUCCESS(rc))
3326	{
3327	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3328	if (RT_SUCCESS(rc))
3329	{
3330	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3331	if (RT_SUCCESS(rc))
3332	{ /* likely */ }
3333	else
3334	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3335	}
3336	else
3337	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3338	}
3339	else
3340	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3341	}
3342	else
3343	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3344	}
3345	else
3346	{
3347	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3348	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3349	if (RT_SUCCESS(rc))
3350	{ /* likely */ }
3351	else
3352	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3353	#else
3354	AssertFailed();
3355	#endif
3356	}
3357	}
3358	else
3359	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3360	}
3361	else
3362	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3363
3364	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3365	if (RT_SUCCESS(rc))
3366	{
3367	rc = hmR0VmxClearVmcs(pVmcsInfo);
3368	if (RT_SUCCESS(rc))
3369	{ /* likely */ }
3370	else
3371	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3372	}
3373
3374	/*
3375	* Update the last-error record both for failures and success, so we
3376	* can propagate the status code back to ring-3 for diagnostics.
3377	*/
3378	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3379	NOREF(pszVmcs);
3380	return rc;
3381	}
3382
3383
3384	/**
3385	* Does global VT-x initialization (called during module initialization).
3386	*
3387	* @returns VBox status code.
3388	*/
3389	VMMR0DECL(int) VMXR0GlobalInit(void)
3390	{
3391	#ifdef HMVMX_USE_FUNCTION_TABLE
3392	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3393	# ifdef VBOX_STRICT
3394	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3395	Assert(g_apfnVMExitHandlers[i]);
3396	# endif
3397	#endif
3398	return VINF_SUCCESS;
3399	}
3400
3401
3402	/**
3403	* Does global VT-x termination (called during module termination).
3404	*/
3405	VMMR0DECL(void) VMXR0GlobalTerm()
3406	{
3407	/* Nothing to do currently. */
3408	}
3409
3410
3411	/**
3412	* Sets up and activates VT-x on the current CPU.
3413	*
3414	* @returns VBox status code.
3415	* @param pHostCpu The HM physical-CPU structure.
3416	* @param pVM The cross context VM structure. Can be
3417	* NULL after a host resume operation.
3418	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3419	* fEnabledByHost is @c true).
3420	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3421	* @a fEnabledByHost is @c true).
3422	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3423	* enable VT-x on the host.
3424	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3425	*/
3426	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3427	PCSUPHWVIRTMSRS pHwvirtMsrs)
3428	{
3429	Assert(pHostCpu);
3430	Assert(pHwvirtMsrs);
3431	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3432
3433	/* Enable VT-x if it's not already enabled by the host. */
3434	if (!fEnabledByHost)
3435	{
3436	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3437	if (RT_FAILURE(rc))
3438	return rc;
3439	}
3440
3441	/*
3442	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3443	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3444	* invalidated when flushing by VPID.
3445	*/
3446	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3447	{
3448	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3449	pHostCpu->fFlushAsidBeforeUse = false;
3450	}
3451	else
3452	pHostCpu->fFlushAsidBeforeUse = true;
3453
3454	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3455	++pHostCpu->cTlbFlushes;
3456
3457	return VINF_SUCCESS;
3458	}
3459
3460
3461	/**
3462	* Deactivates VT-x on the current CPU.
3463	*
3464	* @returns VBox status code.
3465	* @param pvCpuPage Pointer to the VMXON region.
3466	* @param HCPhysCpuPage Physical address of the VMXON region.
3467	*
3468	* @remarks This function should never be called when SUPR0EnableVTx() or
3469	* similar was used to enable VT-x on the host.
3470	*/
3471	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3472	{
3473	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3474
3475	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3476	return hmR0VmxLeaveRootMode();
3477	}
3478
3479
3480	/**
3481	* Does per-VM VT-x initialization.
3482	*
3483	* @returns VBox status code.
3484	* @param pVM The cross context VM structure.
3485	*/
3486	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3487	{
3488	LogFlowFunc(("pVM=%p\n", pVM));
3489
3490	int rc = hmR0VmxStructsAlloc(pVM);
3491	if (RT_FAILURE(rc))
3492	{
3493	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3494	return rc;
3495	}
3496
3497	return VINF_SUCCESS;
3498	}
3499
3500
3501	/**
3502	* Does per-VM VT-x termination.
3503	*
3504	* @returns VBox status code.
3505	* @param pVM The cross context VM structure.
3506	*/
3507	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3508	{
3509	LogFlowFunc(("pVM=%p\n", pVM));
3510
3511	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3512	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3513	{
3514	Assert(pVM->hm.s.vmx.pvScratch);
3515	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3516	}
3517	#endif
3518	hmR0VmxStructsFree(pVM);
3519	return VINF_SUCCESS;
3520	}
3521
3522
3523	/**
3524	* Sets up the VM for execution using hardware-assisted VMX.
3525	* This function is only called once per-VM during initialization.
3526	*
3527	* @returns VBox status code.
3528	* @param pVM The cross context VM structure.
3529	*/
3530	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3531	{
3532	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3533	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3534
3535	LogFlowFunc(("pVM=%p\n", pVM));
3536
3537	/*
3538	* At least verify if VMX is enabled, since we can't check if we're in
3539	* VMX root mode or not without causing a #GP.
3540	*/
3541	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3542	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3543	{ /* likely */ }
3544	else
3545	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3546
3547	/*
3548	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3549	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3550	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3551	*/
3552	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3553	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3554	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3555	{
3556	LogRelFunc(("Invalid real-on-v86 state.\n"));
3557	return VERR_INTERNAL_ERROR;
3558	}
3559
3560	/* Initialize these always, see hmR3InitFinalizeR0().*/
3561	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3562	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3563
3564	/* Setup the tagged-TLB flush handlers. */
3565	int rc = hmR0VmxSetupTaggedTlb(pVM);
3566	if (RT_FAILURE(rc))
3567	{
3568	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3569	return rc;
3570	}
3571
3572	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3573	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3574	#if HC_ARCH_BITS == 64
3575	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3576	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3577	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3578	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3579	#endif
3580
3581	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3582	{
3583	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3584	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3585
3586	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3587	if (RT_SUCCESS(rc))
3588	{
3589	#if HC_ARCH_BITS == 32
3590	hmR0VmxInitVmcsReadCache(pVCpu);
3591	#endif
3592	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3593	if (pVM->cpum.ro.GuestFeatures.fVmx)
3594	{
3595	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3596	if (RT_SUCCESS(rc))
3597	{ /* likely */ }
3598	else
3599	{
3600	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3601	return rc;
3602	}
3603	}
3604	#endif
3605	}
3606	else
3607	{
3608	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3609	return rc;
3610	}
3611	}
3612
3613	return VINF_SUCCESS;
3614	}
3615
3616
3617	#if HC_ARCH_BITS == 32
3618	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3619	/**
3620	* Check if guest state allows safe use of 32-bit switcher again.
3621	*
3622	* Segment bases and protected mode structures must be 32-bit addressable
3623	* because the 32-bit switcher will ignore high dword when writing these VMCS
3624	* fields. See @bugref{8432} for details.
3625	*
3626	* @returns true if safe, false if must continue to use the 64-bit switcher.
3627	* @param pCtx Pointer to the guest-CPU context.
3628	*
3629	* @remarks No-long-jump zone!!!
3630	*/
3631	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3632	{
3633	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3634	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3635	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3636	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3637	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3638	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3639	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3640	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3641	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3642	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3643
3644	/* All good, bases are 32-bit. */
3645	return true;
3646	}
3647	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3648
3649	# ifdef VBOX_STRICT
3650	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3651	{
3652	switch (idxField)
3653	{
3654	case VMX_VMCS_GUEST_RIP:
3655	case VMX_VMCS_GUEST_RSP:
3656	case VMX_VMCS_GUEST_SYSENTER_EIP:
3657	case VMX_VMCS_GUEST_SYSENTER_ESP:
3658	case VMX_VMCS_GUEST_GDTR_BASE:
3659	case VMX_VMCS_GUEST_IDTR_BASE:
3660	case VMX_VMCS_GUEST_CS_BASE:
3661	case VMX_VMCS_GUEST_DS_BASE:
3662	case VMX_VMCS_GUEST_ES_BASE:
3663	case VMX_VMCS_GUEST_FS_BASE:
3664	case VMX_VMCS_GUEST_GS_BASE:
3665	case VMX_VMCS_GUEST_SS_BASE:
3666	case VMX_VMCS_GUEST_LDTR_BASE:
3667	case VMX_VMCS_GUEST_TR_BASE:
3668	case VMX_VMCS_GUEST_CR3:
3669	return true;
3670	}
3671	return false;
3672	}
3673
3674	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3675	{
3676	switch (idxField)
3677	{
3678	/* Read-only fields. */
3679	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3680	return true;
3681	}
3682	/* Remaining readable fields should also be writable. */
3683	return hmR0VmxIsValidWriteField(idxField);
3684	}
3685	# endif /* VBOX_STRICT */
3686
3687
3688	/**
3689	* Executes the specified handler in 64-bit mode.
3690	*
3691	* @returns VBox status code (no informational status codes).
3692	* @param pVCpu The cross context virtual CPU structure.
3693	* @param enmOp The operation to perform.
3694	* @param cParams Number of parameters.
3695	* @param paParam Array of 32-bit parameters.
3696	*/
3697	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3698	{
3699	PVM pVM = pVCpu->CTX_SUFF(pVM);
3700	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3701	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3702	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3703	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3704
3705	#ifdef VBOX_STRICT
3706	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3707	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3708
3709	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3710	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3711	#endif
3712
3713	/* Disable interrupts. */
3714	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3715
3716	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3717	RTCPUID idHostCpu = RTMpCpuId();
3718	CPUMR0SetLApic(pVCpu, idHostCpu);
3719	#endif
3720
3721	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3722
3723	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3724	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3725
3726	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3727	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3728	hmR0VmxClearVmcs(pVmcsInfo);
3729
3730	/* Leave VMX root mode and disable VMX. */
3731	VMXDisable();
3732	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3733
3734	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3735	CPUMSetHyperEIP(pVCpu, enmOp);
3736	for (int i = (int)cParams - 1; i >= 0; i--)
3737	CPUMPushHyper(pVCpu, paParam[i]);
3738
3739	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3740
3741	/* Call the switcher. */
3742	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3743	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3744
3745	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3746	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3747
3748	/* Re-enter VMX root mode. */
3749	int rc2 = VMXEnable(HCPhysCpuPage);
3750	if (RT_FAILURE(rc2))
3751	{
3752	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3753	ASMSetFlags(fOldEFlags);
3754	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3755	return rc2;
3756	}
3757
3758	/* Restore the VMCS as the current VMCS. */
3759	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3760	AssertRC(rc2);
3761	Assert(!(ASMGetFlags() & X86_EFL_IF));
3762	ASMSetFlags(fOldEFlags);
3763	return rc;
3764	}
3765
3766
3767	/**
3768	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3769	* supporting 64-bit guests.
3770	*
3771	* @returns VBox status code.
3772	* @param fResume Whether to VMLAUNCH or VMRESUME.
3773	* @param pCtx Pointer to the guest-CPU context.
3774	* @param pCache Pointer to the VMCS batch cache.
3775	* @param pVM The cross context VM structure.
3776	* @param pVCpu The cross context virtual CPU structure.
3777	*/
3778	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3779	{
3780	NOREF(fResume);
3781
3782	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3783	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3784	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3785
3786	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3787	pCache->uPos = 1;
3788	pCache->interPD = PGMGetInterPaeCR3(pVM);
3789	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3790	#endif
3791
3792	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3793	pCache->TestIn.HCPhysCpuPage = 0;
3794	pCache->TestIn.HCPhysVmcs = 0;
3795	pCache->TestIn.pCache = 0;
3796	pCache->TestOut.HCPhysVmcs = 0;
3797	pCache->TestOut.pCache = 0;
3798	pCache->TestOut.pCtx = 0;
3799	pCache->TestOut.eflags = 0;
3800	#else
3801	NOREF(pCache);
3802	#endif
3803
3804	uint32_t aParam[10];
3805	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3806	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3807	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3808	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3809	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3810	aParam[5] = 0;
3811	aParam[6] = VM_RC_ADDR(pVM, pVM);
3812	aParam[7] = 0;
3813	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3814	aParam[9] = 0;
3815
3816	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3817	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3818	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3819	#endif
3820	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3821
3822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3823	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3824	Assert(pCtx->dr[4] == 10);
3825	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3826	#endif
3827
3828	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3829	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3830	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3831	pVmcsInfo->HCPhysVmcs));
3832	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3833	pCache->TestOut.HCPhysVmcs));
3834	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3835	pCache->TestOut.pCache));
3836	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3837	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3838	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3839	pCache->TestOut.pCtx));
3840	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3841	#endif
3842	NOREF(pCtx);
3843	return rc;
3844	}
3845	#endif
3846
3847
3848	/**
3849	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3850	* the VMCS.
3851	*
3852	* @returns VBox status code.
3853	*/
3854	static int hmR0VmxExportHostControlRegs(void)
3855	{
3856	RTCCUINTREG uReg = ASMGetCR0();
3857	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3858	AssertRCReturn(rc, rc);
3859
3860	uReg = ASMGetCR3();
3861	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3862	AssertRCReturn(rc, rc);
3863
3864	uReg = ASMGetCR4();
3865	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3866	AssertRCReturn(rc, rc);
3867	return rc;
3868	}
3869
3870
3871	/**
3872	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3873	* the host-state area in the VMCS.
3874	*
3875	* @returns VBox status code.
3876	* @param pVCpu The cross context virtual CPU structure.
3877	*/
3878	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3879	{
3880	#if HC_ARCH_BITS == 64
3881	/**
3882	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3883	* requirements. See hmR0VmxExportHostSegmentRegs().
3884	*/
3885	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3886	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3887	{ \
3888	bool fValidSelector = true; \
3889	if ((selValue) & X86_SEL_LDT) \
3890	{ \
3891	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3892	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3893	} \
3894	if (fValidSelector) \
3895	{ \
3896	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3897	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3898	} \
3899	(selValue) = 0; \
3900	}
3901
3902	/*
3903	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3904	* will be messed up. We should -not- save the messed up state without restoring
3905	* the original host-state, see @bugref{7240}.
3906	*
3907	* This apparently can happen (most likely the FPU changes), deal with it rather than
3908	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3909	*/
3910	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3911	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3912	{
3913	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3914	pVCpu->idCpu));
3915	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3916	}
3917	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3918	#else
3919	RT_NOREF(pVCpu);
3920	#endif
3921
3922	/*
3923	* Host DS, ES, FS and GS segment registers.
3924	*/
3925	#if HC_ARCH_BITS == 64
3926	RTSEL uSelDS = ASMGetDS();
3927	RTSEL uSelES = ASMGetES();
3928	RTSEL uSelFS = ASMGetFS();
3929	RTSEL uSelGS = ASMGetGS();
3930	#else
3931	RTSEL uSelDS = 0;
3932	RTSEL uSelES = 0;
3933	RTSEL uSelFS = 0;
3934	RTSEL uSelGS = 0;
3935	#endif
3936
3937	/*
3938	* Host CS and SS segment registers.
3939	*/
3940	RTSEL uSelCS = ASMGetCS();
3941	RTSEL uSelSS = ASMGetSS();
3942
3943	/*
3944	* Host TR segment register.
3945	*/
3946	RTSEL uSelTR = ASMGetTR();
3947
3948	#if HC_ARCH_BITS == 64
3949	/*
3950	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3951	* gain VM-entry and restore them before we get preempted.
3952	*
3953	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3954	*/
3955	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3956	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3957	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3958	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3959	# undef VMXLOCAL_ADJUST_HOST_SEG
3960	#endif
3961
3962	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3963	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3964	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3965	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3966	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3967	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3968	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3969	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3970	Assert(uSelCS);
3971	Assert(uSelTR);
3972
3973	/* Write these host selector fields into the host-state area in the VMCS. */
3974	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3975	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3976	#if HC_ARCH_BITS == 64
3977	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3979	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3980	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3981	#else
3982	NOREF(uSelDS);
3983	NOREF(uSelES);
3984	NOREF(uSelFS);
3985	NOREF(uSelGS);
3986	#endif
3987	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3988	AssertRCReturn(rc, rc);
3989
3990	/*
3991	* Host GDTR and IDTR.
3992	*/
3993	RTGDTR Gdtr;
3994	RTIDTR Idtr;
3995	RT_ZERO(Gdtr);
3996	RT_ZERO(Idtr);
3997	ASMGetGDTR(&Gdtr);
3998	ASMGetIDTR(&Idtr);
3999	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4000	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4001	AssertRCReturn(rc, rc);
4002
4003	#if HC_ARCH_BITS == 64
4004	/*
4005	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4006	* them to the maximum limit (0xffff) on every VM-exit.
4007	*/
4008	if (Gdtr.cbGdt != 0xffff)
4009	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4010
4011	/*
4012	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4013	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4014	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4015	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4016	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4017	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4018	* at 0xffff on hosts where we are sure it won't cause trouble.
4019	*/
4020	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4021	if (Idtr.cbIdt < 0x0fff)
4022	# else
4023	if (Idtr.cbIdt != 0xffff)
4024	# endif
4025	{
4026	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4027	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4028	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4029	}
4030	#endif
4031
4032	/*
4033	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4034	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4035	* RPL should be too in most cases.
4036	*/
4037	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4038	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4039
4040	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4041	#if HC_ARCH_BITS == 64
4042	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4043
4044	/*
4045	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4046	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4047	* restoration if the host has something else. Task switching is not supported in 64-bit
4048	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4049	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4050	*
4051	* [1] See Intel spec. 3.5 "System Descriptor Types".
4052	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4053	*/
4054	PVM pVM = pVCpu->CTX_SUFF(pVM);
4055	Assert(pDesc->System.u4Type == 11);
4056	if ( pDesc->System.u16LimitLow != 0x67
4057	\|\| pDesc->System.u4LimitHigh)
4058	{
4059	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4060	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4061	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4062	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4063	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4064	}
4065
4066	/*
4067	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4068	*/
4069	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4070	{
4071	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4072	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4073	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4074	{
4075	/* The GDT is read-only but the writable GDT is available. */
4076	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4077	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4078	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4079	AssertRCReturn(rc, rc);
4080	}
4081	}
4082	#else
4083	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4084	#endif
4085	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4086	AssertRCReturn(rc, rc);
4087
4088	/*
4089	* Host FS base and GS base.
4090	*/
4091	#if HC_ARCH_BITS == 64
4092	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4093	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4094	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4095	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4096	AssertRCReturn(rc, rc);
4097
4098	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4099	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4100	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4101	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4102	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4103	#endif
4104	return VINF_SUCCESS;
4105	}
4106
4107
4108	/**
4109	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4110	* host-state area of the VMCS.
4111	*
4112	* These MSRs will be automatically restored on the host after every successful
4113	* VM-exit.
4114	*
4115	* @returns VBox status code.
4116	* @param pVCpu The cross context virtual CPU structure.
4117	*
4118	* @remarks No-long-jump zone!!!
4119	*/
4120	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4121	{
4122	AssertPtr(pVCpu);
4123
4124	/*
4125	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4126	* rather than swapping them on every VM-entry.
4127	*/
4128	hmR0VmxLazySaveHostMsrs(pVCpu);
4129
4130	/*
4131	* Host Sysenter MSRs.
4132	*/
4133	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4134	#if HC_ARCH_BITS == 32
4135	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4136	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4137	#else
4138	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4139	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4140	#endif
4141	AssertRCReturn(rc, rc);
4142
4143	/*
4144	* Host EFER MSR.
4145	*
4146	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4147	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4148	*/
4149	PVM pVM = pVCpu->CTX_SUFF(pVM);
4150	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4151	{
4152	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4153	AssertRCReturn(rc, rc);
4154	}
4155
4156	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4157	* hmR0VmxExportGuestEntryExitCtls(). */
4158
4159	return VINF_SUCCESS;
4160	}
4161
4162
4163	/**
4164	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4165	*
4166	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4167	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4168	*
4169	* @returns true if we need to load guest EFER, false otherwise.
4170	* @param pVCpu The cross context virtual CPU structure.
4171	*
4172	* @remarks Requires EFER, CR4.
4173	* @remarks No-long-jump zone!!!
4174	*/
4175	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4176	{
4177	#ifdef HMVMX_ALWAYS_SWAP_EFER
4178	RT_NOREF(pVCpu);
4179	return true;
4180	#else
4181	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4182	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4183	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4184	if (CPUMIsGuestInLongModeEx(pCtx))
4185	return false;
4186	#endif
4187
4188	PVM pVM = pVCpu->CTX_SUFF(pVM);
4189	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4190	uint64_t const u64GuestEfer = pCtx->msrEFER;
4191
4192	/*
4193	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4194	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4195	*/
4196	if ( CPUMIsGuestInLongModeEx(pCtx)
4197	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4198	return true;
4199
4200	/*
4201	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4202	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4203	*
4204	* See Intel spec. 4.5 "IA-32e Paging".
4205	* See Intel spec. 4.1.1 "Three Paging Modes".
4206	*
4207	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4208	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4209	*/
4210	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4211	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4212	if ( (pCtx->cr4 & X86_CR4_PAE)
4213	&& (pCtx->cr0 & X86_CR0_PG)
4214	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4215	{
4216	/* Assert that host is NX capable. */
4217	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4218	return true;
4219	}
4220
4221	return false;
4222	#endif
4223	}
4224
4225	/**
4226	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4227	* VMCS.
4228	*
4229	* This is typically required when the guest changes paging mode.
4230	*
4231	* @returns VBox status code.
4232	* @param pVCpu The cross context virtual CPU structure.
4233	* @param pVmxTransient The VMX-transient structure.
4234	*
4235	* @remarks Requires EFER.
4236	* @remarks No-long-jump zone!!!
4237	*/
4238	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4239	{
4240	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4241	{
4242	PVM pVM = pVCpu->CTX_SUFF(pVM);
4243	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4244
4245	/*
4246	* VM-entry controls.
4247	*/
4248	{
4249	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4250	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4251
4252	/*
4253	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4254	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4255	*
4256	* For nested-guests, this is a mandatory VM-entry control. It's also
4257	* required because we do not want to leak host bits to the nested-guest.
4258	*/
4259	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4260
4261	/*
4262	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4263	*
4264	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4265	* required to get the nested-guest working with hardware-assisted VMX execution.
4266	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4267	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4268	* here rather than while merging the guest VMCS controls.
4269	*/
4270	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4271	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4272	else
4273	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4274
4275	/*
4276	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4277	*
4278	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4279	* regardless of whether the nested-guest VMCS specifies it because we are free to
4280	* load whatever MSRs we require and we do not need to modify the guest visible copy
4281	* of the VM-entry MSR load area.
4282	*/
4283	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4284	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4285	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4286	else
4287	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4288
4289	/*
4290	* The following should -not- be set (since we're not in SMM mode):
4291	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4292	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4293	*/
4294
4295	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4296	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4297
4298	if ((fVal & fZap) == fVal)
4299	{ /* likely */ }
4300	else
4301	{
4302	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4303	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4304	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4305	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4306	}
4307
4308	/* Commit it to the VMCS. */
4309	if (pVmcsInfo->u32EntryCtls != fVal)
4310	{
4311	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4312	AssertRCReturn(rc, rc);
4313	pVmcsInfo->u32EntryCtls = fVal;
4314	}
4315	}
4316
4317	/*
4318	* VM-exit controls.
4319	*/
4320	{
4321	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4322	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4323
4324	/*
4325	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4326	* supported the 1-setting of this bit.
4327	*
4328	* For nested-guests, we set the "save debug controls" as the converse
4329	* "load debug controls" is mandatory for nested-guests anyway.
4330	*/
4331	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4332
4333	/*
4334	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4335	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4336	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4337	* hmR0VmxExportHostMsrs().
4338	*
4339	* For nested-guests, we always set this bit as we do not support 32-bit
4340	* hosts.
4341	*/
4342	#if HC_ARCH_BITS == 64
4343	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4344	#else
4345	Assert(!pVmxTransient->fIsNestedGuest);
4346	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4347	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4348	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4349	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4350	{
4351	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4352	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4353	}
4354	else
4355	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4356	#endif
4357
4358	/*
4359	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4360	*
4361	* For nested-guests, we should use the "save IA32_EFER" control if we also
4362	* used the "load IA32_EFER" control while exporting VM-entry controls.
4363	*/
4364	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4365	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4366	{
4367	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4368	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4369	}
4370
4371	/*
4372	* Enable saving of the VMX-preemption timer value on VM-exit.
4373	* For nested-guests, currently not exposed/used.
4374	*/
4375	if ( pVM->hm.s.vmx.fUsePreemptTimer
4376	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4377	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4378
4379	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4380	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4381
4382	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4383	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4384	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4385
4386	if ((fVal & fZap) == fVal)
4387	{ /* likely */ }
4388	else
4389	{
4390	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4391	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4392	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4393	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4394	}
4395
4396	/* Commit it to the VMCS. */
4397	if (pVmcsInfo->u32ExitCtls != fVal)
4398	{
4399	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4400	AssertRCReturn(rc, rc);
4401	pVmcsInfo->u32ExitCtls = fVal;
4402	}
4403	}
4404
4405	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4406	}
4407	return VINF_SUCCESS;
4408	}
4409
4410
4411	/**
4412	* Sets the TPR threshold in the VMCS.
4413	*
4414	* @returns VBox status code.
4415	* @param pVCpu The cross context virtual CPU structure.
4416	* @param pVmcsInfo The VMCS info. object.
4417	* @param u32TprThreshold The TPR threshold (task-priority class only).
4418	*/
4419	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4420	{
4421	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4422	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4423	RT_NOREF2(pVCpu, pVmcsInfo);
4424	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4425	}
4426
4427
4428	/**
4429	* Exports the guest APIC TPR state into the VMCS.
4430	*
4431	* @returns VBox status code.
4432	* @param pVCpu The cross context virtual CPU structure.
4433	* @param pVmxTransient The VMX-transient structure.
4434	*
4435	* @remarks No-long-jump zone!!!
4436	*/
4437	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4438	{
4439	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4440	{
4441	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4442
4443	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4444	if (!pVmxTransient->fIsNestedGuest)
4445	{
4446	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4447	&& APICIsEnabled(pVCpu))
4448	{
4449	/*
4450	* Setup TPR shadowing.
4451	*/
4452	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4453	{
4454	bool fPendingIntr = false;
4455	uint8_t u8Tpr = 0;
4456	uint8_t u8PendingIntr = 0;
4457	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4458	AssertRCReturn(rc, rc);
4459
4460	/*
4461	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4462	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4463	* priority of the pending interrupt so we can deliver the interrupt. If there
4464	* are no interrupts pending, set threshold to 0 to not cause any
4465	* TPR-below-threshold VM-exits.
4466	*/
4467	Assert(pVmcsInfo->pbVirtApic);
4468	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4469	uint32_t u32TprThreshold = 0;
4470	if (fPendingIntr)
4471	{
4472	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4473	(which is the Task-Priority Class). */
4474	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4475	const uint8_t u8TprPriority = u8Tpr >> 4;
4476	if (u8PendingPriority <= u8TprPriority)
4477	u32TprThreshold = u8PendingPriority;
4478	}
4479
4480	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4481	AssertRCReturn(rc, rc);
4482	}
4483	}
4484	}
4485	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4486	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4487	}
4488	return VINF_SUCCESS;
4489	}
4490
4491
4492	/**
4493	* Gets the guest interruptibility-state.
4494	*
4495	* @returns Guest's interruptibility-state.
4496	* @param pVCpu The cross context virtual CPU structure.
4497	* @param pVmcsInfo The VMCS info. object.
4498	*
4499	* @remarks No-long-jump zone!!!
4500	*/
4501	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4502	{
4503	/*
4504	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4505	*/
4506	uint32_t fIntrState = 0;
4507	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4508	{
4509	/* If inhibition is active, RIP and RFLAGS should've been updated
4510	(i.e. read previously from the VMCS or from ring-3). */
4511	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4512	#ifdef VBOX_STRICT
4513	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4514	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4515	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4516	#endif
4517	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4518	{
4519	if (pCtx->eflags.Bits.u1IF)
4520	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4521	else
4522	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4523	}
4524	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4525	{
4526	/*
4527	* We can clear the inhibit force flag as even if we go back to the recompiler
4528	* without executing guest code in VT-x, the flag's condition to be cleared is
4529	* met and thus the cleared state is correct.
4530	*/
4531	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4532	}
4533	}
4534
4535	/*
4536	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4537	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4538	* setting this would block host-NMIs and IRET will not clear the blocking.
4539	*
4540	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4541	*
4542	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4543	*/
4544	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4545	&& CPUMIsGuestNmiBlocking(pVCpu))
4546	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4547
4548	return fIntrState;
4549	}
4550
4551
4552	/**
4553	* Exports the exception intercepts required for guest execution in the VMCS.
4554	*
4555	* @returns VBox status code.
4556	* @param pVCpu The cross context virtual CPU structure.
4557	* @param pVmxTransient The VMX-transient structure.
4558	*
4559	* @remarks No-long-jump zone!!!
4560	*/
4561	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4562	{
4563	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4564	{
4565	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4566	if ( !pVmxTransient->fIsNestedGuest
4567	&& pVCpu->hm.s.fGIMTrapXcptUD)
4568	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4569	else
4570	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4571
4572	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4573	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4574	}
4575	return VINF_SUCCESS;
4576	}
4577
4578
4579	/**
4580	* Exports the guest's RIP into the guest-state area in the VMCS.
4581	*
4582	* @returns VBox status code.
4583	* @param pVCpu The cross context virtual CPU structure.
4584	*
4585	* @remarks No-long-jump zone!!!
4586	*/
4587	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4588	{
4589	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4590	{
4591	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4592
4593	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4594	AssertRCReturn(rc, rc);
4595
4596	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4597	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4598	}
4599	return VINF_SUCCESS;
4600	}
4601
4602
4603	/**
4604	* Exports the guest's RSP into the guest-state area in the VMCS.
4605	*
4606	* @returns VBox status code.
4607	* @param pVCpu The cross context virtual CPU structure.
4608	*
4609	* @remarks No-long-jump zone!!!
4610	*/
4611	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4612	{
4613	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4614	{
4615	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4616
4617	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4618	AssertRCReturn(rc, rc);
4619
4620	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4621	}
4622	return VINF_SUCCESS;
4623	}
4624
4625
4626	/**
4627	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4628	*
4629	* @returns VBox status code.
4630	* @param pVCpu The cross context virtual CPU structure.
4631	* @param pVmxTransient The VMX-transient structure.
4632	*
4633	* @remarks No-long-jump zone!!!
4634	*/
4635	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4636	{
4637	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4638	{
4639	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4640
4641	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4642	Let us assert it as such and use 32-bit VMWRITE. */
4643	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4644	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4645	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4646	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4647
4648	/*
4649	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4650	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4651	* can run the real-mode guest code under Virtual 8086 mode.
4652	*/
4653	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4654	if (pVmcsInfo->RealMode.fRealOnV86Active)
4655	{
4656	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4657	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4658	Assert(!pVmxTransient->fIsNestedGuest);
4659	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4660	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4661	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4662	}
4663
4664	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4665	AssertRCReturn(rc, rc);
4666
4667	/*
4668	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4669	*
4670	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4671	* through the hypervisor debugger using EFLAGS.TF.
4672	*/
4673	if ( !pVmxTransient->fIsNestedGuest
4674	&& !pVCpu->hm.s.fSingleInstruction
4675	&& fEFlags.Bits.u1TF)
4676	{
4677	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4678	* premature trips to ring-3 esp since IEM does not yet handle it. */
4679	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4680	AssertRCReturn(rc, rc);
4681	}
4682	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4683	* nested-guest VMCS. */
4684
4685	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4686	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4687	}
4688	return VINF_SUCCESS;
4689	}
4690
4691
4692	/**
4693	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4694	*
4695	* The guest FPU state is always pre-loaded hence we don't need to bother about
4696	* sharing FPU related CR0 bits between the guest and host.
4697	*
4698	* @returns VBox status code.
4699	* @param pVCpu The cross context virtual CPU structure.
4700	* @param pVmxTransient The VMX-transient structure.
4701	*
4702	* @remarks No-long-jump zone!!!
4703	*/
4704	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4705	{
4706	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4707	{
4708	PVM pVM = pVCpu->CTX_SUFF(pVM);
4709	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4710
4711	/*
4712	* Figure out fixed CR0 bits in VMX operation.
4713	*/
4714	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4715	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4716	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4717	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4718	else
4719	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4720
4721	if (!pVmxTransient->fIsNestedGuest)
4722	{
4723	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4724	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4725	uint64_t const u64ShadowCr0 = u64GuestCr0;
4726	Assert(!RT_HI_U32(u64GuestCr0));
4727
4728	/*
4729	* Setup VT-x's view of the guest CR0.
4730	*/
4731	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4732	if (pVM->hm.s.fNestedPaging)
4733	{
4734	if (CPUMIsGuestPagingEnabled(pVCpu))
4735	{
4736	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4737	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4738	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4739	}
4740	else
4741	{
4742	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4743	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4744	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4745	}
4746
4747	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4748	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4749	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4750	}
4751	else
4752	{
4753	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4754	u64GuestCr0 \|= X86_CR0_WP;
4755	}
4756
4757	/*
4758	* Guest FPU bits.
4759	*
4760	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4761	* using CR0.TS.
4762	*
4763	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4764	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4765	*/
4766	u64GuestCr0 \|= X86_CR0_NE;
4767
4768	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4769	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4770
4771	/*
4772	* Update exception intercepts.
4773	*/
4774	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4775	if (pVmcsInfo->RealMode.fRealOnV86Active)
4776	{
4777	Assert(PDMVmmDevHeapIsEnabled(pVM));
4778	Assert(pVM->hm.s.vmx.pRealModeTSS);
4779	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4780	}
4781	else
4782	{
4783	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4784	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4785	if (fInterceptMF)
4786	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4787	}
4788
4789	/* Additional intercepts for debugging, define these yourself explicitly. */
4790	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4791	uXcptBitmap \|= 0
4792	\| RT_BIT(X86_XCPT_BP)
4793	\| RT_BIT(X86_XCPT_DE)
4794	\| RT_BIT(X86_XCPT_NM)
4795	\| RT_BIT(X86_XCPT_TS)
4796	\| RT_BIT(X86_XCPT_UD)
4797	\| RT_BIT(X86_XCPT_NP)
4798	\| RT_BIT(X86_XCPT_SS)
4799	\| RT_BIT(X86_XCPT_GP)
4800	\| RT_BIT(X86_XCPT_PF)
4801	\| RT_BIT(X86_XCPT_MF)
4802	;
4803	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4804	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4805	#endif
4806	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4807	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4808	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4809
4810	/* Apply the fixed CR0 bits and enable caching. */
4811	u64GuestCr0 \|= fSetCr0;
4812	u64GuestCr0 &= fZapCr0;
4813	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4814
4815	/* Commit the CR0 and related fields to the guest VMCS. */
4816	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
4817	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4818	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4819	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4820	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4821	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4822	AssertRCReturn(rc, rc);
4823
4824	/* Update our caches. */
4825	pVmcsInfo->u32ProcCtls = uProcCtls;
4826	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4827
4828	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4829	}
4830	else
4831	{
4832	/*
4833	* With nested-guests, we may have extended the guest/host mask here (since we
4834	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
4835	* mask can include more bits (to read from the nested-guest CR0 read-shadow) than
4836	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
4837	* those bits from the nested-guest CR0 into the nested-guest CR0 read shadow.
4838	*/
4839	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4840	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4841	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx);
4842	Assert(!RT_HI_U32(u64GuestCr0));
4843	Assert(u64GuestCr0 & X86_CR0_NE);
4844
4845	/* Apply the fixed CR0 bits and enable caching. */
4846	u64GuestCr0 \|= fSetCr0;
4847	u64GuestCr0 &= fZapCr0;
4848	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4849
4850	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4851	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
4852	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4853	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4854	AssertRCReturn(rc, rc);
4855
4856	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4857	}
4858
4859	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4860	}
4861
4862	return VINF_SUCCESS;
4863	}
4864
4865
4866	/**
4867	* Exports the guest control registers (CR3, CR4) into the guest-state area
4868	* in the VMCS.
4869	*
4870	* @returns VBox strict status code.
4871	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4872	* without unrestricted guest access and the VMMDev is not presently
4873	* mapped (e.g. EFI32).
4874	*
4875	* @param pVCpu The cross context virtual CPU structure.
4876	* @param pVmxTransient The VMX-transient structure.
4877	*
4878	* @remarks No-long-jump zone!!!
4879	*/
4880	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4881	{
4882	int rc = VINF_SUCCESS;
4883	PVM pVM = pVCpu->CTX_SUFF(pVM);
4884
4885	/*
4886	* Guest CR2.
4887	* It's always loaded in the assembler code. Nothing to do here.
4888	*/
4889
4890	/*
4891	* Guest CR3.
4892	*/
4893	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4894	{
4895	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4896
4897	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4898	if (pVM->hm.s.fNestedPaging)
4899	{
4900	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4901	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4902
4903	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4904	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4905	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4906	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4907
4908	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4909	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4910	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4911
4912	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4913	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4914	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4915	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4916	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4917	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4918	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4919
4920	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4921	AssertRCReturn(rc, rc);
4922
4923	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4924	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4925	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4926	{
4927	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4928	if (CPUMIsGuestInPAEModeEx(pCtx))
4929	{
4930	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4931	AssertRCReturn(rc, rc);
4932	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4933	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4934	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4935	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4936	AssertRCReturn(rc, rc);
4937	}
4938
4939	/*
4940	* The guest's view of its CR3 is unblemished with nested paging when the
4941	* guest is using paging or we have unrestricted guest execution to handle
4942	* the guest when it's not using paging.
4943	*/
4944	GCPhysGuestCR3 = pCtx->cr3;
4945	}
4946	else
4947	{
4948	/*
4949	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4950	* thinks it accesses physical memory directly, we use our identity-mapped
4951	* page table to map guest-linear to guest-physical addresses. EPT takes care
4952	* of translating it to host-physical addresses.
4953	*/
4954	RTGCPHYS GCPhys;
4955	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4956
4957	/* We obtain it here every time as the guest could have relocated this PCI region. */
4958	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4959	if (RT_SUCCESS(rc))
4960	{ /* likely */ }
4961	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4962	{
4963	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4964	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4965	}
4966	else
4967	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4968
4969	GCPhysGuestCR3 = GCPhys;
4970	}
4971
4972	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4973	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4974	AssertRCReturn(rc, rc);
4975	}
4976	else
4977	{
4978	/* Non-nested paging case, just use the hypervisor's CR3. */
4979	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4980
4981	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4982	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4983	AssertRCReturn(rc, rc);
4984	}
4985
4986	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4987	}
4988
4989	/*
4990	* Guest CR4.
4991	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4992	*/
4993	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4994	{
4995	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4996	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4997
4998	/*
4999	* Figure out fixed CR4 bits in VMX operation.
5000	*/
5001	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5002	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5003
5004	/*
5005	* With nested-guests, we may have extended the guest/host mask here (since we
5006	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5007	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5008	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5009	* those bits from the nested-guest CR4 into the nested-guest CR4 read shadow.
5010	*/
5011	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5012	uint64_t u64GuestCr4 = pCtx->cr4;
5013	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5014	? pCtx->cr4
5015	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx);
5016	Assert(!RT_HI_U32(u64GuestCr4));
5017
5018	/*
5019	* Setup VT-x's view of the guest CR4.
5020	*
5021	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5022	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5023	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5024	*
5025	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5026	*/
5027	if (pVmcsInfo->RealMode.fRealOnV86Active)
5028	{
5029	Assert(pVM->hm.s.vmx.pRealModeTSS);
5030	Assert(PDMVmmDevHeapIsEnabled(pVM));
5031	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5032	}
5033
5034	if (pVM->hm.s.fNestedPaging)
5035	{
5036	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5037	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5038	{
5039	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5040	u64GuestCr4 \|= X86_CR4_PSE;
5041	/* Our identity mapping is a 32-bit page directory. */
5042	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5043	}
5044	/* else use guest CR4.*/
5045	}
5046	else
5047	{
5048	Assert(!pVmxTransient->fIsNestedGuest);
5049
5050	/*
5051	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5052	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5053	*/
5054	switch (pVCpu->hm.s.enmShadowMode)
5055	{
5056	case PGMMODE_REAL: /* Real-mode. */
5057	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5058	case PGMMODE_32_BIT: /* 32-bit paging. */
5059	{
5060	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5061	break;
5062	}
5063
5064	case PGMMODE_PAE: /* PAE paging. */
5065	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5066	{
5067	u64GuestCr4 \|= X86_CR4_PAE;
5068	break;
5069	}
5070
5071	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5072	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5073	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5074	break;
5075	#endif
5076	default:
5077	AssertFailed();
5078	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5079	}
5080	}
5081
5082	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5083	u64GuestCr4 \|= fSetCr4;
5084	u64GuestCr4 &= fZapCr4;
5085
5086	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5087	/** @todo Fix to 64-bit when we drop 32-bit. */
5088	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5089	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5090	AssertRCReturn(rc, rc);
5091
5092	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5093	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5094
5095	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5096
5097	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5098	}
5099	return rc;
5100	}
5101
5102
5103	/**
5104	* Exports the guest debug registers into the guest-state area in the VMCS.
5105	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5106	*
5107	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5108	*
5109	* @returns VBox status code.
5110	* @param pVCpu The cross context virtual CPU structure.
5111	* @param pVmxTransient The VMX-transient structure.
5112	*
5113	* @remarks No-long-jump zone!!!
5114	*/
5115	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5116	{
5117	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5118
5119	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5120	* stepping. */
5121	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5122	if (pVmxTransient->fIsNestedGuest)
5123	{
5124	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5125	AssertRCReturn(rc, rc);
5126	return VINF_SUCCESS;
5127	}
5128
5129	#ifdef VBOX_STRICT
5130	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5131	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5132	{
5133	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5134	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5135	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5136	}
5137	#endif
5138
5139	bool fSteppingDB = false;
5140	bool fInterceptMovDRx = false;
5141	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5142	if (pVCpu->hm.s.fSingleInstruction)
5143	{
5144	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5145	PVM pVM = pVCpu->CTX_SUFF(pVM);
5146	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5147	{
5148	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5149	Assert(fSteppingDB == false);
5150	}
5151	else
5152	{
5153	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5154	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5155	pVCpu->hm.s.fClearTrapFlag = true;
5156	fSteppingDB = true;
5157	}
5158	}
5159
5160	uint32_t u32GuestDr7;
5161	if ( fSteppingDB
5162	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5163	{
5164	/*
5165	* Use the combined guest and host DRx values found in the hypervisor register set
5166	* because the hypervisor debugger has breakpoints active or someone is single stepping
5167	* on the host side without a monitor trap flag.
5168	*
5169	* Note! DBGF expects a clean DR6 state before executing guest code.
5170	*/
5171	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5172	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5173	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5174	{
5175	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5176	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5177	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5178	}
5179	else
5180	#endif
5181	if (!CPUMIsHyperDebugStateActive(pVCpu))
5182	{
5183	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5184	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5185	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5186	}
5187
5188	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5189	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5190	pVCpu->hm.s.fUsingHyperDR7 = true;
5191	fInterceptMovDRx = true;
5192	}
5193	else
5194	{
5195	/*
5196	* If the guest has enabled debug registers, we need to load them prior to
5197	* executing guest code so they'll trigger at the right time.
5198	*/
5199	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5200	{
5201	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5202	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5203	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5204	{
5205	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5206	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5207	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5208	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5209	}
5210	else
5211	#endif
5212	if (!CPUMIsGuestDebugStateActive(pVCpu))
5213	{
5214	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5215	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5216	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5217	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5218	}
5219	Assert(!fInterceptMovDRx);
5220	}
5221	/*
5222	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5223	* must intercept #DB in order to maintain a correct DR6 guest value, and
5224	* because we need to intercept it to prevent nested #DBs from hanging the
5225	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5226	*/
5227	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5228	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5229	&& !CPUMIsGuestDebugStateActive(pVCpu))
5230	#else
5231	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5232	#endif
5233	{
5234	fInterceptMovDRx = true;
5235	}
5236
5237	/* Update DR7 with the actual guest value. */
5238	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5239	pVCpu->hm.s.fUsingHyperDR7 = false;
5240	}
5241
5242	if (fInterceptMovDRx)
5243	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5244	else
5245	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5246
5247	/*
5248	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5249	* monitor-trap flag and update our cache.
5250	*/
5251	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5252	{
5253	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5254	AssertRCReturn(rc2, rc2);
5255	pVmcsInfo->u32ProcCtls = uProcCtls;
5256	}
5257
5258	/*
5259	* Update guest DR7.
5260	*/
5261	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5262	AssertRCReturn(rc, rc);
5263
5264	/*
5265	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5266	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5267	*
5268	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5269	*/
5270	if (fSteppingDB)
5271	{
5272	Assert(pVCpu->hm.s.fSingleInstruction);
5273	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5274
5275	uint32_t fIntrState = 0;
5276	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5277	AssertRCReturn(rc, rc);
5278
5279	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5280	{
5281	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5282	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5283	AssertRCReturn(rc, rc);
5284	}
5285	}
5286
5287	return VINF_SUCCESS;
5288	}
5289
5290
5291	#ifdef VBOX_STRICT
5292	/**
5293	* Strict function to validate segment registers.
5294	*
5295	* @param pVCpu The cross context virtual CPU structure.
5296	* @param pVmcsInfo The VMCS info. object.
5297	*
5298	* @remarks Will import guest CR0 on strict builds during validation of
5299	* segments.
5300	*/
5301	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5302	{
5303	/*
5304	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5305	*
5306	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5307	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5308	* unusable bit and doesn't change the guest-context value.
5309	*/
5310	PVM pVM = pVCpu->CTX_SUFF(pVM);
5311	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5312	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5313	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5314	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5315	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5316	{
5317	/* Protected mode checks */
5318	/* CS */
5319	Assert(pCtx->cs.Attr.n.u1Present);
5320	Assert(!(pCtx->cs.Attr.u & 0xf00));
5321	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5322	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5323	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5324	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5325	\|\| (pCtx->cs.Attr.n.u1Granularity));
5326	/* CS cannot be loaded with NULL in protected mode. */
5327	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5328	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5329	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5330	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5331	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5332	else
5333	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5334	/* SS */
5335	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5336	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5337	if ( !(pCtx->cr0 & X86_CR0_PE)
5338	\|\| pCtx->cs.Attr.n.u4Type == 3)
5339	{
5340	Assert(!pCtx->ss.Attr.n.u2Dpl);
5341	}
5342	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5343	{
5344	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5345	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5346	Assert(pCtx->ss.Attr.n.u1Present);
5347	Assert(!(pCtx->ss.Attr.u & 0xf00));
5348	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5349	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5350	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5351	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5352	\|\| (pCtx->ss.Attr.n.u1Granularity));
5353	}
5354	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5355	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5356	{
5357	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5358	Assert(pCtx->ds.Attr.n.u1Present);
5359	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5360	Assert(!(pCtx->ds.Attr.u & 0xf00));
5361	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5362	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5363	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5364	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5365	\|\| (pCtx->ds.Attr.n.u1Granularity));
5366	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5367	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5368	}
5369	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5370	{
5371	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5372	Assert(pCtx->es.Attr.n.u1Present);
5373	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5374	Assert(!(pCtx->es.Attr.u & 0xf00));
5375	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5376	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5377	\|\| !(pCtx->es.Attr.n.u1Granularity));
5378	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5379	\|\| (pCtx->es.Attr.n.u1Granularity));
5380	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5381	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5382	}
5383	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5384	{
5385	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5386	Assert(pCtx->fs.Attr.n.u1Present);
5387	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5388	Assert(!(pCtx->fs.Attr.u & 0xf00));
5389	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5390	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5391	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5392	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5393	\|\| (pCtx->fs.Attr.n.u1Granularity));
5394	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5395	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5396	}
5397	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5398	{
5399	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5400	Assert(pCtx->gs.Attr.n.u1Present);
5401	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5402	Assert(!(pCtx->gs.Attr.u & 0xf00));
5403	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5404	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5405	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5406	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5407	\|\| (pCtx->gs.Attr.n.u1Granularity));
5408	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5409	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5410	}
5411	/* 64-bit capable CPUs. */
5412	# if HC_ARCH_BITS == 64
5413	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5414	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5415	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5416	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5417	# endif
5418	}
5419	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5420	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5421	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5422	{
5423	/* Real and v86 mode checks. */
5424	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5425	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5426	if (pVmcsInfo->RealMode.fRealOnV86Active)
5427	{
5428	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5429	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5430	}
5431	else
5432	{
5433	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5434	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5435	}
5436
5437	/* CS */
5438	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5439	Assert(pCtx->cs.u32Limit == 0xffff);
5440	Assert(u32CSAttr == 0xf3);
5441	/* SS */
5442	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5443	Assert(pCtx->ss.u32Limit == 0xffff);
5444	Assert(u32SSAttr == 0xf3);
5445	/* DS */
5446	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5447	Assert(pCtx->ds.u32Limit == 0xffff);
5448	Assert(u32DSAttr == 0xf3);
5449	/* ES */
5450	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5451	Assert(pCtx->es.u32Limit == 0xffff);
5452	Assert(u32ESAttr == 0xf3);
5453	/* FS */
5454	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5455	Assert(pCtx->fs.u32Limit == 0xffff);
5456	Assert(u32FSAttr == 0xf3);
5457	/* GS */
5458	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5459	Assert(pCtx->gs.u32Limit == 0xffff);
5460	Assert(u32GSAttr == 0xf3);
5461	/* 64-bit capable CPUs. */
5462	# if HC_ARCH_BITS == 64
5463	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5464	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5465	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5466	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5467	# endif
5468	}
5469	}
5470	#endif /* VBOX_STRICT */
5471
5472
5473	/**
5474	* Exports a guest segment register into the guest-state area in the VMCS.
5475	*
5476	* @returns VBox status code.
5477	* @param pVCpu The cross context virtual CPU structure.
5478	* @param pVmcsInfo The VMCS info. object.
5479	* @param iSegReg The segment register number (X86_SREG_XXX).
5480	* @param pSelReg Pointer to the segment selector.
5481	*
5482	* @remarks No-long-jump zone!!!
5483	*/
5484	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5485	{
5486	Assert(iSegReg < X86_SREG_COUNT);
5487	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5488	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5489	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5490	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5491
5492	uint32_t u32Access = pSelReg->Attr.u;
5493	if (pVmcsInfo->RealMode.fRealOnV86Active)
5494	{
5495	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5496	u32Access = 0xf3;
5497	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5498	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5499	RT_NOREF_PV(pVCpu);
5500	}
5501	else
5502	{
5503	/*
5504	* The way to differentiate between whether this is really a null selector or was just
5505	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5506	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5507	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5508	* NULL selectors loaded in protected-mode have their attribute as 0.
5509	*/
5510	if (!u32Access)
5511	u32Access = X86DESCATTR_UNUSABLE;
5512	}
5513
5514	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5515	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5516	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5517
5518	/*
5519	* Commit it to the VMCS.
5520	*/
5521	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5522	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5523	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5524	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5525	AssertRCReturn(rc, rc);
5526	return rc;
5527	}
5528
5529
5530	/**
5531	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5532	* area in the VMCS.
5533	*
5534	* @returns VBox status code.
5535	* @param pVCpu The cross context virtual CPU structure.
5536	* @param pVmxTransient The VMX-transient structure.
5537	*
5538	* @remarks Will import guest CR0 on strict builds during validation of
5539	* segments.
5540	* @remarks No-long-jump zone!!!
5541	*/
5542	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5543	{
5544	int rc = VERR_INTERNAL_ERROR_5;
5545	PVM pVM = pVCpu->CTX_SUFF(pVM);
5546	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5547	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5548
5549	/*
5550	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5551	*/
5552	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5553	{
5554	#ifdef VBOX_WITH_REM
5555	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5556	{
5557	Assert(!pVmxTransient->fIsNestedGuest);
5558	Assert(pVM->hm.s.vmx.pRealModeTSS);
5559	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5560	if ( pVmcsInfo->fWasInRealMode
5561	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5562	{
5563	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5564	in real-mode (e.g. OpenBSD 4.0) */
5565	REMFlushTBs(pVM);
5566	Log4Func(("Switch to protected mode detected!\n"));
5567	pVmcsInfo->fWasInRealMode = false;
5568	}
5569	}
5570	#endif
5571	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5572	{
5573	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5574	if (pVmcsInfo->RealMode.fRealOnV86Active)
5575	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5576	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5577	AssertRCReturn(rc, rc);
5578	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5579	}
5580
5581	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5582	{
5583	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5584	if (pVmcsInfo->RealMode.fRealOnV86Active)
5585	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5586	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5587	AssertRCReturn(rc, rc);
5588	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5589	}
5590
5591	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5592	{
5593	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5594	if (pVmcsInfo->RealMode.fRealOnV86Active)
5595	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5596	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5597	AssertRCReturn(rc, rc);
5598	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5599	}
5600
5601	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5602	{
5603	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5604	if (pVmcsInfo->RealMode.fRealOnV86Active)
5605	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5606	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5607	AssertRCReturn(rc, rc);
5608	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5609	}
5610
5611	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5612	{
5613	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5614	if (pVmcsInfo->RealMode.fRealOnV86Active)
5615	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5616	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5617	AssertRCReturn(rc, rc);
5618	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5619	}
5620
5621	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5622	{
5623	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5624	if (pVmcsInfo->RealMode.fRealOnV86Active)
5625	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5626	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5627	AssertRCReturn(rc, rc);
5628	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5629	}
5630
5631	#ifdef VBOX_STRICT
5632	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5633	#endif
5634	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5635	pCtx->cs.Attr.u));
5636	}
5637
5638	/*
5639	* Guest TR.
5640	*/
5641	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5642	{
5643	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5644
5645	/*
5646	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5647	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5648	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5649	*/
5650	uint16_t u16Sel;
5651	uint32_t u32Limit;
5652	uint64_t u64Base;
5653	uint32_t u32AccessRights;
5654	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5655	{
5656	u16Sel = pCtx->tr.Sel;
5657	u32Limit = pCtx->tr.u32Limit;
5658	u64Base = pCtx->tr.u64Base;
5659	u32AccessRights = pCtx->tr.Attr.u;
5660	}
5661	else
5662	{
5663	Assert(!pVmxTransient->fIsNestedGuest);
5664	Assert(pVM->hm.s.vmx.pRealModeTSS);
5665	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5666
5667	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5668	RTGCPHYS GCPhys;
5669	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5670	AssertRCReturn(rc, rc);
5671
5672	X86DESCATTR DescAttr;
5673	DescAttr.u = 0;
5674	DescAttr.n.u1Present = 1;
5675	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5676
5677	u16Sel = 0;
5678	u32Limit = HM_VTX_TSS_SIZE;
5679	u64Base = GCPhys;
5680	u32AccessRights = DescAttr.u;
5681	}
5682
5683	/* Validate. */
5684	Assert(!(u16Sel & RT_BIT(2)));
5685	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5686	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5687	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5688	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5689	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5690	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5691	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5692	Assert( (u32Limit & 0xfff) == 0xfff
5693	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5694	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5695	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5696
5697	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5698	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5699	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5700	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5701	AssertRCReturn(rc, rc);
5702
5703	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5704	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5705	}
5706
5707	/*
5708	* Guest GDTR.
5709	*/
5710	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5711	{
5712	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5713
5714	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5715	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5716	AssertRCReturn(rc, rc);
5717
5718	/* Validate. */
5719	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5720
5721	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5722	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5723	}
5724
5725	/*
5726	* Guest LDTR.
5727	*/
5728	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5729	{
5730	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5731
5732	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5733	uint32_t u32Access;
5734	if ( !pVmxTransient->fIsNestedGuest
5735	&& !pCtx->ldtr.Attr.u)
5736	u32Access = X86DESCATTR_UNUSABLE;
5737	else
5738	u32Access = pCtx->ldtr.Attr.u;
5739
5740	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5741	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5742	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5743	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5744	AssertRCReturn(rc, rc);
5745
5746	/* Validate. */
5747	if (!(u32Access & X86DESCATTR_UNUSABLE))
5748	{
5749	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5750	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5751	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5752	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5753	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5754	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5755	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5756	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5757	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5758	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5759	}
5760
5761	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5762	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5763	}
5764
5765	/*
5766	* Guest IDTR.
5767	*/
5768	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5769	{
5770	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5771
5772	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5773	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5774	AssertRCReturn(rc, rc);
5775
5776	/* Validate. */
5777	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5778
5779	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5780	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5781	}
5782
5783	return VINF_SUCCESS;
5784	}
5785
5786
5787	/**
5788	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5789	* areas.
5790	*
5791	* These MSRs will automatically be loaded to the host CPU on every successful
5792	* VM-entry and stored from the host CPU on every successful VM-exit.
5793	*
5794	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5795	* actual host MSR values are not- updated here for performance reasons. See
5796	* hmR0VmxExportHostMsrs().
5797	*
5798	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5799	*
5800	* @returns VBox status code.
5801	* @param pVCpu The cross context virtual CPU structure.
5802	* @param pVmxTransient The VMX-transient structure.
5803	*
5804	* @remarks No-long-jump zone!!!
5805	*/
5806	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5807	{
5808	AssertPtr(pVCpu);
5809	AssertPtr(pVmxTransient);
5810
5811	PVM pVM = pVCpu->CTX_SUFF(pVM);
5812	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5813
5814	/*
5815	* MSRs that we use the auto-load/store MSR area in the VMCS.
5816	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5817	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5818	*
5819	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5820	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5821	* emulation, nothing to do here.
5822	*/
5823	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5824	{
5825	if ( !pVmxTransient->fIsNestedGuest
5826	&& pVM->hm.s.fAllow64BitGuests)
5827	{
5828	#if HC_ARCH_BITS == 32
5829	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5830	Assert(!pVmxTransient->fIsNestedGuest);
5831
5832	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5833	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5834	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5835	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5836	AssertRCReturn(rc, rc);
5837	#endif
5838	}
5839	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5840	}
5841
5842	/*
5843	* Guest Sysenter MSRs.
5844	*/
5845	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5846	{
5847	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5848
5849	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5850	{
5851	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5852	AssertRCReturn(rc, rc);
5853	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5854	}
5855
5856	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5857	{
5858	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5859	AssertRCReturn(rc, rc);
5860	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5861	}
5862
5863	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5864	{
5865	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5866	AssertRCReturn(rc, rc);
5867	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5868	}
5869	}
5870
5871	/*
5872	* Guest/host EFER MSR.
5873	*/
5874	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5875	{
5876	/* Whether we are using the VMCS to swap the EFER MSR must have been
5877	determined earlier while exporting VM-entry/VM-exit controls. */
5878	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5879	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5880
5881	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5882	{
5883	/*
5884	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5885	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5886	*/
5887	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5888	{
5889	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5890	AssertRCReturn(rc, rc);
5891	}
5892	else
5893	{
5894	/*
5895	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5896	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5897	*/
5898	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5899	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5900	AssertRCReturn(rc, rc);
5901	}
5902	}
5903	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5904	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5905
5906	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5907	}
5908
5909	/*
5910	* Other MSRs.
5911	* Speculation Control (R/W).
5912	*/
5913	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5914	{
5915	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5916	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5917	{
5918	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5919	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5920	AssertRCReturn(rc, rc);
5921	}
5922	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5923	}
5924
5925	return VINF_SUCCESS;
5926	}
5927
5928
5929	/**
5930	* Selects up the appropriate function to run guest code.
5931	*
5932	* @returns VBox status code.
5933	* @param pVCpu The cross context virtual CPU structure.
5934	* @param pVmxTransient The VMX-transient structure.
5935	*
5936	* @remarks No-long-jump zone!!!
5937	*/
5938	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5939	{
5940	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5941	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5942
5943	if (CPUMIsGuestInLongModeEx(pCtx))
5944	{
5945	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5946	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5947	#endif
5948	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5949	#if HC_ARCH_BITS == 32
5950	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5951	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5952	{
5953	#ifdef VBOX_STRICT
5954	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5955	{
5956	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5957	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5958	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5959	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5960	("fCtxChanged=%#RX64\n", fCtxChanged));
5961	}
5962	#endif
5963	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5964
5965	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5966	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5967	pVmcsInfo->fSwitchedTo64on32 = true;
5968	Log4Func(("Selected 64-bit switcher\n"));
5969	}
5970	#else
5971	/* 64-bit host. */
5972	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5973	#endif
5974	}
5975	else
5976	{
5977	/* Guest is not in long mode, use the 32-bit handler. */
5978	#if HC_ARCH_BITS == 32
5979	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5980	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5981	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5982	{
5983	# ifdef VBOX_STRICT
5984	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5985	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5986	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5987	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5988	("fCtxChanged=%#RX64\n", fCtxChanged));
5989	# endif
5990	}
5991	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5992	/*
5993	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5994	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5995	* switcher flag now because we know the guest is in a sane state where it's safe
5996	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5997	* the much faster 32-bit switcher again.
5998	*/
5999	if (!pVmcsInfo->fSwitchedTo64on32)
6000	{
6001	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6002	Log4Func(("Selected 32-bit switcher\n"));
6003	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6004	}
6005	else
6006	{
6007	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6008	if ( pVmcsInfo->RealMode.fRealOnV86Active
6009	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6010	{
6011	pVmcsInfo->fSwitchedTo64on32 = false;
6012	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6014	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6015	\| HM_CHANGED_HOST_CONTEXT);
6016	Log4Func(("Selected 32-bit switcher (safe)\n"));
6017	}
6018	}
6019	# else
6020	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6021	# endif
6022	#else
6023	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6024	#endif
6025	}
6026	Assert(pVmcsInfo->pfnStartVM);
6027	return VINF_SUCCESS;
6028	}
6029
6030
6031	/**
6032	* Wrapper for running the guest code in VT-x.
6033	*
6034	* @returns VBox status code, no informational status codes.
6035	* @param pVCpu The cross context virtual CPU structure.
6036	* @param pVmxTransient The VMX-transient structure.
6037	*
6038	* @remarks No-long-jump zone!!!
6039	*/
6040	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6041	{
6042	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6043	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6044	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6045
6046	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6047
6048	/*
6049	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6050	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6051	* callee-saved and thus the need for this XMM wrapper.
6052	*
6053	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6054	*/
6055	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6056	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6057	PVM pVM = pVCpu->CTX_SUFF(pVM);
6058	#ifdef VBOX_WITH_KERNEL_USING_XMM
6059	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6060	#else
6061	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6062	#endif
6063	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6064	return rc;
6065	}
6066
6067
6068	/**
6069	* Reports world-switch error and dumps some useful debug info.
6070	*
6071	* @param pVCpu The cross context virtual CPU structure.
6072	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6073	* @param pVmxTransient The VMX-transient structure (only
6074	* exitReason updated).
6075	*/
6076	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6077	{
6078	Assert(pVCpu);
6079	Assert(pVmxTransient);
6080	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6081
6082	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6083	switch (rcVMRun)
6084	{
6085	case VERR_VMX_INVALID_VMXON_PTR:
6086	AssertFailed();
6087	break;
6088	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6089	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6090	{
6091	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6092	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6093	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6094	AssertRC(rc);
6095
6096	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6097	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6098	Cannot do it here as we may have been long preempted. */
6099
6100	#ifdef VBOX_STRICT
6101	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6102	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6103	pVmxTransient->uExitReason));
6104	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6105	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6106	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6107	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6108	else
6109	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6110	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6111	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6112
6113	/* VMX control bits. */
6114	uint32_t u32Val;
6115	uint64_t u64Val;
6116	RTHCUINTREG uHCReg;
6117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6118	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6120	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6121	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6122	{
6123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6124	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6125	}
6126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6127	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6129	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6131	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6132	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6133	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6135	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6136	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6137	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6138	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6139	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6140	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6141	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6142	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6143	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6144	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6145	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6146	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6147	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6148	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6149	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6150	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6151	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6152	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6153	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6154	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6155	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6156	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6157	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6158	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6159	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6160	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6161	{
6162	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6163	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6164	}
6165
6166	/* Guest bits. */
6167	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6168	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6169	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6170	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6171	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6172	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6173	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6174	{
6175	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6176	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6177	}
6178
6179	/* Host bits. */
6180	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6181	Log4(("Host CR0 %#RHr\n", uHCReg));
6182	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6183	Log4(("Host CR3 %#RHr\n", uHCReg));
6184	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6185	Log4(("Host CR4 %#RHr\n", uHCReg));
6186
6187	RTGDTR HostGdtr;
6188	PCX86DESCHC pDesc;
6189	ASMGetGDTR(&HostGdtr);
6190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6191	Log4(("Host CS %#08x\n", u32Val));
6192	if (u32Val < HostGdtr.cbGdt)
6193	{
6194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6195	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6196	}
6197
6198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6199	Log4(("Host DS %#08x\n", u32Val));
6200	if (u32Val < HostGdtr.cbGdt)
6201	{
6202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6203	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6204	}
6205
6206	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6207	Log4(("Host ES %#08x\n", u32Val));
6208	if (u32Val < HostGdtr.cbGdt)
6209	{
6210	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6211	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6212	}
6213
6214	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6215	Log4(("Host FS %#08x\n", u32Val));
6216	if (u32Val < HostGdtr.cbGdt)
6217	{
6218	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6219	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6220	}
6221
6222	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6223	Log4(("Host GS %#08x\n", u32Val));
6224	if (u32Val < HostGdtr.cbGdt)
6225	{
6226	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6227	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6228	}
6229
6230	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6231	Log4(("Host SS %#08x\n", u32Val));
6232	if (u32Val < HostGdtr.cbGdt)
6233	{
6234	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6235	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6236	}
6237
6238	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6239	Log4(("Host TR %#08x\n", u32Val));
6240	if (u32Val < HostGdtr.cbGdt)
6241	{
6242	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6243	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6244	}
6245
6246	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6247	Log4(("Host TR Base %#RHv\n", uHCReg));
6248	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6249	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6250	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6251	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6252	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6253	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6254	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6255	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6256	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6257	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6258	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6259	Log4(("Host RSP %#RHv\n", uHCReg));
6260	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6261	Log4(("Host RIP %#RHv\n", uHCReg));
6262	# if HC_ARCH_BITS == 64
6263	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6264	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6265	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6266	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6267	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6268	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6269	# endif
6270	#endif /* VBOX_STRICT */
6271	break;
6272	}
6273
6274	default:
6275	/* Impossible */
6276	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6277	break;
6278	}
6279	}
6280
6281
6282	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6283	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6284	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6285	# endif
6286
6287	/**
6288	* Initialize the VMCS-Read cache.
6289	*
6290	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6291	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6292	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6293	* (those that have a 32-bit FULL & HIGH part).
6294	*
6295	* @param pVCpu The cross context virtual CPU structure.
6296	*/
6297	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6298	{
6299	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6300	do { \
6301	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6302	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6303	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6304	++cReadFields; \
6305	} while (0)
6306
6307	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6308	uint32_t cReadFields = 0;
6309
6310	/*
6311	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6312	* and serve to indicate exceptions to the rules.
6313	*/
6314
6315	/* Guest-natural selector base fields. */
6316	#if 0
6317	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6320	#endif
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6333	#if 0
6334	/* Unused natural width guest-state fields. */
6335	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6336	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6337	#endif
6338	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6339	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6340
6341	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6342	these 64-bit fields (using "FULL" and "HIGH" fields). */
6343	#if 0
6344	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6345	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6346	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6347	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6348	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6349	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6350	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6351	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6352	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6353	#endif
6354
6355	/* Natural width guest-state fields. */
6356	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6357	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6358
6359	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6360	{
6361	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6362	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6363	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6364	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6365	}
6366	else
6367	{
6368	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6369	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6370	}
6371
6372	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6373	}
6374
6375
6376	/**
6377	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6378	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6379	* darwin, running 64-bit guests).
6380	*
6381	* @returns VBox status code.
6382	* @param pVCpu The cross context virtual CPU structure.
6383	* @param idxField The VMCS field encoding.
6384	* @param u64Val 16, 32 or 64-bit value.
6385	*/
6386	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6387	{
6388	int rc;
6389	switch (idxField)
6390	{
6391	/*
6392	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6393	*/
6394	/* 64-bit Control fields. */
6395	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6396	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6397	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6398	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6399	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6400	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6401	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6402	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6403	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6404	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6405	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6406	case VMX_VMCS64_CTRL_EPTP_FULL:
6407	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6408	/* 64-bit Guest-state fields. */
6409	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6410	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6411	case VMX_VMCS64_GUEST_PAT_FULL:
6412	case VMX_VMCS64_GUEST_EFER_FULL:
6413	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6414	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6415	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6416	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6417	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6418	/* 64-bit Host-state fields. */
6419	case VMX_VMCS64_HOST_PAT_FULL:
6420	case VMX_VMCS64_HOST_EFER_FULL:
6421	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6422	{
6423	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6424	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6425	break;
6426	}
6427
6428	/*
6429	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6430	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6431	*/
6432	/* Natural-width Guest-state fields. */
6433	case VMX_VMCS_GUEST_CR3:
6434	case VMX_VMCS_GUEST_ES_BASE:
6435	case VMX_VMCS_GUEST_CS_BASE:
6436	case VMX_VMCS_GUEST_SS_BASE:
6437	case VMX_VMCS_GUEST_DS_BASE:
6438	case VMX_VMCS_GUEST_FS_BASE:
6439	case VMX_VMCS_GUEST_GS_BASE:
6440	case VMX_VMCS_GUEST_LDTR_BASE:
6441	case VMX_VMCS_GUEST_TR_BASE:
6442	case VMX_VMCS_GUEST_GDTR_BASE:
6443	case VMX_VMCS_GUEST_IDTR_BASE:
6444	case VMX_VMCS_GUEST_RSP:
6445	case VMX_VMCS_GUEST_RIP:
6446	case VMX_VMCS_GUEST_SYSENTER_ESP:
6447	case VMX_VMCS_GUEST_SYSENTER_EIP:
6448	{
6449	if (!(RT_HI_U32(u64Val)))
6450	{
6451	/* If this field is 64-bit, VT-x will zero out the top bits. */
6452	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6453	}
6454	else
6455	{
6456	/* Assert that only the 32->64 switcher case should ever come here. */
6457	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6458	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6459	}
6460	break;
6461	}
6462
6463	default:
6464	{
6465	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6466	pVCpu->hm.s.u32HMError = idxField;
6467	rc = VERR_INVALID_PARAMETER;
6468	break;
6469	}
6470	}
6471	AssertRCReturn(rc, rc);
6472	return rc;
6473	}
6474
6475
6476	/**
6477	* Queue up a VMWRITE by using the VMCS write cache.
6478	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6479	*
6480	* @param pVCpu The cross context virtual CPU structure.
6481	* @param idxField The VMCS field encoding.
6482	* @param u64Val 16, 32 or 64-bit value.
6483	*/
6484	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6485	{
6486	AssertPtr(pVCpu);
6487	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6488
6489	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6490	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6491
6492	/* Make sure there are no duplicates. */
6493	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6494	{
6495	if (pCache->Write.aField[i] == idxField)
6496	{
6497	pCache->Write.aFieldVal[i] = u64Val;
6498	return VINF_SUCCESS;
6499	}
6500	}
6501
6502	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6503	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6504	pCache->Write.cValidEntries++;
6505	return VINF_SUCCESS;
6506	}
6507	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6508
6509
6510	/**
6511	* Sets up the usage of TSC-offsetting and updates the VMCS.
6512	*
6513	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6514	* VMX-preemption timer.
6515	*
6516	* @returns VBox status code.
6517	* @param pVCpu The cross context virtual CPU structure.
6518	* @param pVmxTransient The VMX-transient structure.
6519	*
6520	* @remarks No-long-jump zone!!!
6521	*/
6522	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6523	{
6524	bool fOffsettedTsc;
6525	bool fParavirtTsc;
6526	uint64_t uTscOffset;
6527	PVM pVM = pVCpu->CTX_SUFF(pVM);
6528	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6529
6530	if (pVM->hm.s.vmx.fUsePreemptTimer)
6531	{
6532	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6533
6534	/* Make sure the returned values have sane upper and lower boundaries. */
6535	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6536	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6537	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6538	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6539
6540	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6541	* preemption timers here. We probably need to clamp the preemption timer,
6542	* after converting the timer value to the host. */
6543	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6544	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6545	AssertRC(rc);
6546	}
6547	else
6548	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6549
6550	if (fParavirtTsc)
6551	{
6552	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6553	information before every VM-entry, hence disable it for performance sake. */
6554	#if 0
6555	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6556	AssertRC(rc);
6557	#endif
6558	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6559	}
6560
6561	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6562	if ( fOffsettedTsc
6563	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6564	{
6565	if (pVmxTransient->fIsNestedGuest)
6566	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6567	if (pVmcsInfo->u64TscOffset != uTscOffset)
6568	{
6569	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6570	AssertRC(rc);
6571	pVmcsInfo->u64TscOffset = uTscOffset;
6572	}
6573
6574	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6575	{
6576	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6577	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6578	AssertRC(rc);
6579	pVmcsInfo->u32ProcCtls = uProcCtls;
6580	}
6581	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6582	}
6583	else
6584	{
6585	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6586	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6587	{
6588	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6589	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6590	AssertRC(rc);
6591	pVmcsInfo->u32ProcCtls = uProcCtls;
6592	}
6593	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6594	}
6595	}
6596
6597
6598	/**
6599	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6600	* VM-exit interruption info type.
6601	*
6602	* @returns The IEM exception flags.
6603	* @param uVector The event vector.
6604	* @param uVmxEventType The VMX event type.
6605	*
6606	* @remarks This function currently only constructs flags required for
6607	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6608	* and CR2 aspects of an exception are not included).
6609	*/
6610	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6611	{
6612	uint32_t fIemXcptFlags;
6613	switch (uVmxEventType)
6614	{
6615	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6616	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6617	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6618	break;
6619
6620	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6621	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6622	break;
6623
6624	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6625	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6626	break;
6627
6628	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6629	{
6630	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6631	if (uVector == X86_XCPT_BP)
6632	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6633	else if (uVector == X86_XCPT_OF)
6634	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6635	else
6636	{
6637	fIemXcptFlags = 0;
6638	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6639	}
6640	break;
6641	}
6642
6643	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6644	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6645	break;
6646
6647	default:
6648	fIemXcptFlags = 0;
6649	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6650	break;
6651	}
6652	return fIemXcptFlags;
6653	}
6654
6655
6656	/**
6657	* Sets an event as a pending event to be injected into the guest.
6658	*
6659	* @param pVCpu The cross context virtual CPU structure.
6660	* @param u32IntInfo The VM-entry interruption-information field.
6661	* @param cbInstr The VM-entry instruction length in bytes (for software
6662	* interrupts, exceptions and privileged software
6663	* exceptions).
6664	* @param u32ErrCode The VM-entry exception error code.
6665	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6666	* page-fault.
6667	*/
6668	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6669	RTGCUINTPTR GCPtrFaultAddress)
6670	{
6671	Assert(!pVCpu->hm.s.Event.fPending);
6672	pVCpu->hm.s.Event.fPending = true;
6673	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6674	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6675	pVCpu->hm.s.Event.cbInstr = cbInstr;
6676	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6677	}
6678
6679
6680	/**
6681	* Sets an external interrupt as pending-for-injection into the VM.
6682	*
6683	* @param pVCpu The cross context virtual CPU structure.
6684	* @param u8Interrupt The external interrupt vector.
6685	*/
6686	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6687	{
6688	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6689	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6690	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6691	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6692	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6693	}
6694
6695
6696	/**
6697	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6698	*
6699	* @param pVCpu The cross context virtual CPU structure.
6700	*/
6701	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6702	{
6703	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6704	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6706	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6707	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6708	}
6709
6710
6711	/**
6712	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6713	*
6714	* @param pVCpu The cross context virtual CPU structure.
6715	*/
6716	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6717	{
6718	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6719	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6720	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6721	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6722	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6723	}
6724
6725
6726	/**
6727	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6728	*
6729	* @param pVCpu The cross context virtual CPU structure.
6730	*/
6731	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6732	{
6733	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6734	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6735	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6736	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6737	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6738	}
6739
6740
6741	/**
6742	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6743	*
6744	* @param pVCpu The cross context virtual CPU structure.
6745	*/
6746	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6747	{
6748	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6750	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6751	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6752	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6753	}
6754
6755
6756	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6757	/**
6758	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6759	*
6760	* @param pVCpu The cross context virtual CPU structure.
6761	* @param u32ErrCode The error code for the general-protection exception.
6762	*/
6763	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6764	{
6765	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6766	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6767	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6768	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6769	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6770	}
6771
6772
6773	/**
6774	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6775	*
6776	* @param pVCpu The cross context virtual CPU structure.
6777	* @param u32ErrCode The error code for the stack exception.
6778	*/
6779	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6780	{
6781	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6782	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6783	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6784	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6785	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6786	}
6787
6788
6789	/**
6790	* Decodes the memory operand of an instruction that caused a VM-exit.
6791	*
6792	* The VM-exit qualification field provides the displacement field for memory
6793	* operand instructions, if any.
6794	*
6795	* @returns Strict VBox status code (i.e. informational status codes too).
6796	* @retval VINF_SUCCESS if the operand was successfully decoded.
6797	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6798	* operand.
6799	* @param pVCpu The cross context virtual CPU structure.
6800	* @param uExitInstrInfo The VM-exit instruction information field.
6801	* @param enmMemAccess The memory operand's access type (read or write).
6802	* @param GCPtrDisp The instruction displacement field, if any. For
6803	* RIP-relative addressing pass RIP + displacement here.
6804	* @param pGCPtrMem Where to store the effective destination memory address.
6805	*
6806	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
6807	* virtual-8086 mode hence skips those checks while verifying if the
6808	* segment is valid.
6809	*/
6810	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6811	PRTGCPTR pGCPtrMem)
6812	{
6813	Assert(pGCPtrMem);
6814	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6815	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6816	\| CPUMCTX_EXTRN_CR0);
6817
6818	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6819	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6820	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6821
6822	VMXEXITINSTRINFO ExitInstrInfo;
6823	ExitInstrInfo.u = uExitInstrInfo;
6824	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6825	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6826	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6827	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6828	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6829	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6830	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6831	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6832	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6833
6834	/*
6835	* Validate instruction information.
6836	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6837	*/
6838	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6839	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6840	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6841	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6842	AssertLogRelMsgReturn(fIsMemOperand,
6843	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6844
6845	/*
6846	* Compute the complete effective address.
6847	*
6848	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6849	* See AMD spec. 4.5.2 "Segment Registers".
6850	*/
6851	RTGCPTR GCPtrMem = GCPtrDisp;
6852	if (fBaseRegValid)
6853	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6854	if (fIdxRegValid)
6855	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6856
6857	RTGCPTR const GCPtrOff = GCPtrMem;
6858	if ( !fIsLongMode
6859	\|\| iSegReg >= X86_SREG_FS)
6860	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6861	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6862
6863	/*
6864	* Validate effective address.
6865	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6866	*/
6867	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6868	Assert(cbAccess > 0);
6869	if (fIsLongMode)
6870	{
6871	if (X86_IS_CANONICAL(GCPtrMem))
6872	{
6873	*pGCPtrMem = GCPtrMem;
6874	return VINF_SUCCESS;
6875	}
6876
6877	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6878	* "Data Limit Checks in 64-bit Mode". */
6879	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6880	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6881	return VINF_HM_PENDING_XCPT;
6882	}
6883
6884	/*
6885	* This is a watered down version of iemMemApplySegment().
6886	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6887	* and segment CPL/DPL checks are skipped.
6888	*/
6889	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6890	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6891	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6892
6893	/* Check if the segment is present and usable. */
6894	if ( pSel->Attr.n.u1Present
6895	&& !pSel->Attr.n.u1Unusable)
6896	{
6897	Assert(pSel->Attr.n.u1DescType);
6898	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6899	{
6900	/* Check permissions for the data segment. */
6901	if ( enmMemAccess == VMXMEMACCESS_WRITE
6902	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6903	{
6904	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6905	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6906	return VINF_HM_PENDING_XCPT;
6907	}
6908
6909	/* Check limits if it's a normal data segment. */
6910	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6911	{
6912	if ( GCPtrFirst32 > pSel->u32Limit
6913	\|\| GCPtrLast32 > pSel->u32Limit)
6914	{
6915	Log4Func(("Data segment limit exceeded. "
6916	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6917	GCPtrLast32, pSel->u32Limit));
6918	if (iSegReg == X86_SREG_SS)
6919	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6920	else
6921	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6922	return VINF_HM_PENDING_XCPT;
6923	}
6924	}
6925	else
6926	{
6927	/* Check limits if it's an expand-down data segment.
6928	Note! The upper boundary is defined by the B bit, not the G bit! */
6929	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6930	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6931	{
6932	Log4Func(("Expand-down data segment limit exceeded. "
6933	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6934	GCPtrLast32, pSel->u32Limit));
6935	if (iSegReg == X86_SREG_SS)
6936	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6937	else
6938	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6939	return VINF_HM_PENDING_XCPT;
6940	}
6941	}
6942	}
6943	else
6944	{
6945	/* Check permissions for the code segment. */
6946	if ( enmMemAccess == VMXMEMACCESS_WRITE
6947	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6948	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6949	{
6950	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6951	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6952	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6953	return VINF_HM_PENDING_XCPT;
6954	}
6955
6956	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6957	if ( GCPtrFirst32 > pSel->u32Limit
6958	\|\| GCPtrLast32 > pSel->u32Limit)
6959	{
6960	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6961	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6962	if (iSegReg == X86_SREG_SS)
6963	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6964	else
6965	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6966	return VINF_HM_PENDING_XCPT;
6967	}
6968	}
6969	}
6970	else
6971	{
6972	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6973	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6974	return VINF_HM_PENDING_XCPT;
6975	}
6976
6977	*pGCPtrMem = GCPtrMem;
6978	return VINF_SUCCESS;
6979	}
6980
6981
6982	/**
6983	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6984	* guest attempting to execute a VMX instruction.
6985	*
6986	* @returns Strict VBox status code (i.e. informational status codes too).
6987	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6988	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6989	*
6990	* @param pVCpu The cross context virtual CPU structure.
6991	* @param uExitReason The VM-exit reason.
6992	*
6993	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6994	* @remarks No-long-jump zone!!!
6995	*/
6996	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6997	{
6998	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6999	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7000
7001	/*
7002	* The physical CPU would have already checked the CPU mode/code segment.
7003	* We shall just assert here for paranoia.
7004	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7005	*/
7006	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7007	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7008	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7009
7010	if (uExitReason == VMX_EXIT_VMXON)
7011	{
7012	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7013
7014	/*
7015	* We check CR4.VMXE because it is required to be always set while in VMX operation
7016	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
7017	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7018	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7019	*/
7020	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7021	{
7022	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7023	hmR0VmxSetPendingXcptUD(pVCpu);
7024	return VINF_HM_PENDING_XCPT;
7025	}
7026	}
7027	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7028	{
7029	/*
7030	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7031	* (other than VMXON), we need to raise a #UD.
7032	*/
7033	Log4Func(("Not in VMX root mode -> #UD\n"));
7034	hmR0VmxSetPendingXcptUD(pVCpu);
7035	return VINF_HM_PENDING_XCPT;
7036	}
7037
7038	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7039	return VINF_SUCCESS;
7040	}
7041	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7042
7043
7044	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7045	{
7046	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7047
7048	/*
7049	* If VT-x marks the segment as unusable, most other bits remain undefined:
7050	* - For CS the L, D and G bits have meaning.
7051	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7052	* - For the remaining data segments no bits are defined.
7053	*
7054	* The present bit and the unusable bit has been observed to be set at the
7055	* same time (the selector was supposed to be invalid as we started executing
7056	* a V8086 interrupt in ring-0).
7057	*
7058	* What should be important for the rest of the VBox code, is that the P bit is
7059	* cleared. Some of the other VBox code recognizes the unusable bit, but
7060	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7061	* safe side here, we'll strip off P and other bits we don't care about. If
7062	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7063	*
7064	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7065	*/
7066	#ifdef VBOX_STRICT
7067	uint32_t const uAttr = pSelReg->Attr.u;
7068	#endif
7069
7070	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7071	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7072	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7073
7074	#ifdef VBOX_STRICT
7075	VMMRZCallRing3Disable(pVCpu);
7076	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7077	# ifdef DEBUG_bird
7078	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7079	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7080	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7081	# endif
7082	VMMRZCallRing3Enable(pVCpu);
7083	NOREF(uAttr);
7084	#endif
7085	RT_NOREF2(pVCpu, idxSel);
7086	}
7087
7088
7089	/**
7090	* Imports a guest segment register from the current VMCS into the guest-CPU
7091	* context.
7092	*
7093	* @returns VBox status code.
7094	* @param pVCpu The cross context virtual CPU structure.
7095	* @param iSegReg The segment register number (X86_SREG_XXX).
7096	*
7097	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7098	* do not log!
7099	*/
7100	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7101	{
7102	Assert(iSegReg < X86_SREG_COUNT);
7103
7104	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7105	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7106	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7107	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7108	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7109	#else
7110	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7111	#endif
7112	uint64_t u64Base;
7113	uint32_t u32Sel, u32Limit, u32Attr;
7114	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7115	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7116	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7117	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7118	if (RT_SUCCESS(rc))
7119	{
7120	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7121	pSelReg->Sel = u32Sel;
7122	pSelReg->ValidSel = u32Sel;
7123	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7124	pSelReg->u32Limit = u32Limit;
7125	pSelReg->u64Base = u64Base;
7126	pSelReg->Attr.u = u32Attr;
7127	if (u32Attr & X86DESCATTR_UNUSABLE)
7128	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7129	}
7130	return rc;
7131	}
7132
7133
7134	/**
7135	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7136	*
7137	* @returns VBox status code.
7138	* @param pVCpu The cross context virtual CPU structure.
7139	*
7140	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7141	* do not log!
7142	*/
7143	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7144	{
7145	uint64_t u64Base;
7146	uint32_t u32Sel, u32Limit, u32Attr;
7147	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7148	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7149	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7150	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7151	if (RT_SUCCESS(rc))
7152	{
7153	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7154	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7155	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7156	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7157	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7158	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7159	if (u32Attr & X86DESCATTR_UNUSABLE)
7160	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7161	}
7162	return rc;
7163	}
7164
7165
7166	/**
7167	* Imports the guest TR from the current VMCS into the guest-CPU context.
7168	*
7169	* @returns VBox status code.
7170	* @param pVCpu The cross context virtual CPU structure.
7171	*
7172	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7173	* do not log!
7174	*/
7175	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7176	{
7177	uint32_t u32Sel, u32Limit, u32Attr;
7178	uint64_t u64Base;
7179	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7180	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7181	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7182	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7183	AssertRCReturn(rc, rc);
7184
7185	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7186	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7187	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7188	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7189	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7190	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7191	/* TR is the only selector that can never be unusable. */
7192	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7193	return VINF_SUCCESS;
7194	}
7195
7196
7197	/**
7198	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7199	*
7200	* @returns VBox status code.
7201	* @param pVCpu The cross context virtual CPU structure.
7202	*
7203	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7204	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7205	* instead!!!
7206	*/
7207	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7208	{
7209	uint64_t u64Val;
7210	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7211	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7212	{
7213	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7214	if (RT_SUCCESS(rc))
7215	{
7216	pCtx->rip = u64Val;
7217	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7218	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7219	}
7220	return rc;
7221	}
7222	return VINF_SUCCESS;
7223	}
7224
7225
7226	/**
7227	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7228	*
7229	* @returns VBox status code.
7230	* @param pVCpu The cross context virtual CPU structure.
7231	* @param pVmcsInfo The VMCS info. object.
7232	*
7233	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7234	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7235	* instead!!!
7236	*/
7237	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7238	{
7239	uint32_t u32Val;
7240	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7241	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7242	{
7243	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7244	if (RT_SUCCESS(rc))
7245	{
7246	pCtx->eflags.u32 = u32Val;
7247
7248	/* Restore eflags for real-on-v86-mode hack. */
7249	if (pVmcsInfo->RealMode.fRealOnV86Active)
7250	{
7251	pCtx->eflags.Bits.u1VM = 0;
7252	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7253	}
7254	}
7255	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7256	return rc;
7257	}
7258	return VINF_SUCCESS;
7259	}
7260
7261
7262	/**
7263	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7264	* context.
7265	*
7266	* @returns VBox status code.
7267	* @param pVCpu The cross context virtual CPU structure.
7268	* @param pVmcsInfo The VMCS info. object.
7269	*
7270	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7271	* do not log!
7272	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7273	* instead!!!
7274	*/
7275	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7276	{
7277	uint32_t u32Val;
7278	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7279	if (RT_SUCCESS(rc))
7280	{
7281	if (!u32Val)
7282	{
7283	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7284	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7285
7286	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7287	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7288	}
7289	else
7290	{
7291	/*
7292	* We must import RIP here to set our EM interrupt-inhibited state.
7293	* We also import RFLAGS as our code that evaluates pending interrupts
7294	* before VM-entry requires it.
7295	*/
7296	rc = hmR0VmxImportGuestRip(pVCpu);
7297	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7298	if (RT_SUCCESS(rc))
7299	{
7300	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7301	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7302	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7303	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7304
7305	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7306	{
7307	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7308	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7309	}
7310	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7311	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7312	}
7313	}
7314	}
7315	return rc;
7316	}
7317
7318
7319	/**
7320	* Worker for VMXR0ImportStateOnDemand.
7321	*
7322	* @returns VBox status code.
7323	* @param pVCpu The cross context virtual CPU structure.
7324	* @param pVmcsInfo The VMCS info. object.
7325	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7326	*/
7327	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7328	{
7329	#define VMXLOCAL_BREAK_RC(a_rc) \
7330	if (RT_SUCCESS(a_rc)) \
7331	{ } \
7332	else \
7333	break
7334
7335	int rc = VINF_SUCCESS;
7336	PVM pVM = pVCpu->CTX_SUFF(pVM);
7337	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7338	uint64_t u64Val;
7339	uint32_t u32Val;
7340
7341	/*
7342	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7343	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7344	* neither are other host platforms.
7345	*
7346	* Committing this temporarily as it prevents BSOD.
7347	*
7348	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7349	*/
7350	#ifdef RT_OS_WINDOWS
7351	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7352	return VERR_HM_IPE_1;
7353	#endif
7354
7355	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7356
7357	/*
7358	* We disable interrupts to make the updating of the state and in particular
7359	* the fExtrn modification atomic wrt to preemption hooks.
7360	*/
7361	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7362
7363	fWhat &= pCtx->fExtrn;
7364	if (fWhat)
7365	{
7366	do
7367	{
7368	if (fWhat & CPUMCTX_EXTRN_RIP)
7369	{
7370	rc = hmR0VmxImportGuestRip(pVCpu);
7371	VMXLOCAL_BREAK_RC(rc);
7372	}
7373
7374	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7375	{
7376	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7377	VMXLOCAL_BREAK_RC(rc);
7378	}
7379
7380	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7381	{
7382	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7383	VMXLOCAL_BREAK_RC(rc);
7384	}
7385
7386	if (fWhat & CPUMCTX_EXTRN_RSP)
7387	{
7388	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7389	VMXLOCAL_BREAK_RC(rc);
7390	pCtx->rsp = u64Val;
7391	}
7392
7393	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7394	{
7395	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7396	if (fWhat & CPUMCTX_EXTRN_CS)
7397	{
7398	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7399	rc \|= hmR0VmxImportGuestRip(pVCpu);
7400	if (fRealOnV86Active)
7401	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7402	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7403	}
7404	if (fWhat & CPUMCTX_EXTRN_SS)
7405	{
7406	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7407	if (fRealOnV86Active)
7408	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7409	}
7410	if (fWhat & CPUMCTX_EXTRN_DS)
7411	{
7412	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7413	if (fRealOnV86Active)
7414	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7415	}
7416	if (fWhat & CPUMCTX_EXTRN_ES)
7417	{
7418	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7419	if (fRealOnV86Active)
7420	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7421	}
7422	if (fWhat & CPUMCTX_EXTRN_FS)
7423	{
7424	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7425	if (fRealOnV86Active)
7426	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7427	}
7428	if (fWhat & CPUMCTX_EXTRN_GS)
7429	{
7430	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7431	if (fRealOnV86Active)
7432	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7433	}
7434	VMXLOCAL_BREAK_RC(rc);
7435	}
7436
7437	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7438	{
7439	if (fWhat & CPUMCTX_EXTRN_LDTR)
7440	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7441
7442	if (fWhat & CPUMCTX_EXTRN_GDTR)
7443	{
7444	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7445	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7446	pCtx->gdtr.pGdt = u64Val;
7447	pCtx->gdtr.cbGdt = u32Val;
7448	}
7449
7450	/* Guest IDTR. */
7451	if (fWhat & CPUMCTX_EXTRN_IDTR)
7452	{
7453	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7454	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7455	pCtx->idtr.pIdt = u64Val;
7456	pCtx->idtr.cbIdt = u32Val;
7457	}
7458
7459	/* Guest TR. */
7460	if (fWhat & CPUMCTX_EXTRN_TR)
7461	{
7462	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7463	don't need to import that one. */
7464	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7465	rc \|= hmR0VmxImportGuestTr(pVCpu);
7466	}
7467	VMXLOCAL_BREAK_RC(rc);
7468	}
7469
7470	if (fWhat & CPUMCTX_EXTRN_DR7)
7471	{
7472	if (!pVCpu->hm.s.fUsingHyperDR7)
7473	{
7474	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7475	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7476	VMXLOCAL_BREAK_RC(rc);
7477	pCtx->dr[7] = u32Val;
7478	}
7479	}
7480
7481	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7482	{
7483	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7484	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7485	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7486	pCtx->SysEnter.cs = u32Val;
7487	VMXLOCAL_BREAK_RC(rc);
7488	}
7489
7490	#if HC_ARCH_BITS == 64
7491	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7492	{
7493	if ( pVM->hm.s.fAllow64BitGuests
7494	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7495	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7496	}
7497
7498	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7499	{
7500	if ( pVM->hm.s.fAllow64BitGuests
7501	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7502	{
7503	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7504	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7505	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7506	}
7507	}
7508	#endif
7509
7510	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7511	#if HC_ARCH_BITS == 32
7512	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7513	#endif
7514	)
7515	{
7516	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7517	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7518	Assert(pMsrs);
7519	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7520	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7521	for (uint32_t i = 0; i < cMsrs; i++)
7522	{
7523	uint32_t const idMsr = pMsrs[i].u32Msr;
7524	switch (idMsr)
7525	{
7526	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7527	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7528	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7529	#if HC_ARCH_BITS == 32
7530	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7531	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7532	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7533	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7534	#endif
7535	default:
7536	{
7537	pCtx->fExtrn = 0;
7538	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7539	ASMSetFlags(fEFlags);
7540	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7541	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7542	}
7543	}
7544	}
7545	}
7546
7547	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7548	{
7549	uint64_t u64Shadow;
7550	if (fWhat & CPUMCTX_EXTRN_CR0)
7551	{
7552	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7553	* remove when we drop 32-bit host w/ 64-bit host support, see
7554	* @bugref{9180#c39}. */
7555	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7556	#if HC_ARCH_BITS == 32
7557	uint32_t u32Shadow;
7558	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7559	u64Shadow = u32Shadow;
7560	#else
7561	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7562	#endif
7563	VMXLOCAL_BREAK_RC(rc);
7564	u64Val = u32Val;
7565	#if 1
7566	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7567	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7568	#else
7569	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
7570	{
7571	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7572	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7573	}
7574	else
7575	{
7576	/** @todo NSTVMX: We need to do some unfudging here because we altered the
7577	* guest/host mask before running the nested-guest. */
7578	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
7579	Assert(pVmcsNstGst);
7580
7581	uint64_t const uGstCr0Mask = pVmcsNstGst->u64Cr0Mask.u;
7582	uint64_t const uHstCr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
7583	}
7584	#endif
7585	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7586	CPUMSetGuestCR0(pVCpu, u64Val);
7587	VMMRZCallRing3Enable(pVCpu);
7588	}
7589
7590	if (fWhat & CPUMCTX_EXTRN_CR4)
7591	{
7592	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7593	* remove when we drop 32-bit host w/ 64-bit host support, see
7594	* @bugref{9180#c39}. */
7595	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7596	#if HC_ARCH_BITS == 32
7597	uint32_t u32Shadow;
7598	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7599	u64Shadow = u32Shadow;
7600	#else
7601	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7602	#endif
7603	VMXLOCAL_BREAK_RC(rc);
7604	u64Val = u32Val;
7605	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7606	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7607	pCtx->cr4 = u64Val;
7608	}
7609
7610	if (fWhat & CPUMCTX_EXTRN_CR3)
7611	{
7612	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7613	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7614	\|\| ( pVM->hm.s.fNestedPaging
7615	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7616	{
7617	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7618	VMXLOCAL_BREAK_RC(rc);
7619	if (pCtx->cr3 != u64Val)
7620	{
7621	pCtx->cr3 = u64Val;
7622	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7623	}
7624
7625	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7626	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7627	if (CPUMIsGuestInPAEModeEx(pCtx))
7628	{
7629	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7630	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7631	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7632	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7633	VMXLOCAL_BREAK_RC(rc);
7634	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7635	}
7636	}
7637	}
7638
7639	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7640	# if 0
7641	/** @todo NSTVMX: We handle each of these fields individually by passing it to IEM
7642	* VM-exit handlers. We might handle it differently when using the fast path. */
7643	/*
7644	* The hardware virtualization state currently consists of VMCS fields that may be
7645	* modified by execution of the nested-guest (that are not part of the general
7646	* guest state) and is visible to guest software. Hence, it is technically part of
7647	* the guest-CPU state when executing a nested-guest.
7648	*/
7649	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7650	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7651	{
7652	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7653	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7654	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7655	VMXLOCAL_BREAK_RC(rc);
7656
7657	/*
7658	* VM-entry can fail due to invalid-guest state, machine-check events and
7659	* MSR loading failures. Other than VM-exit reason and VM-exit qualification
7660	* all other VMCS fields are left unmodified on VM-entry failure.
7661	*
7662	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7663	*/
7664	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7665	if (!fEntryFailed)
7666	{
7667	/*
7668	* Some notes on VMCS fields that may need importing when the fast path
7669	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7670	*
7671	* Requires fixing up when using hardware-assisted VMX:
7672	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7673	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7674	* - IDT-vectoring info: Think about this.
7675	* - IDT-vectoring error code: Think about this.
7676	*
7677	* Emulated:
7678	* - Guest-interruptiblity state: Derived from FFs and RIP.
7679	* - Guest pending debug exceptions: Derived from DR6.
7680	* - Guest activity state: Emulated from EM state.
7681	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7682	* - Entry-interrupt info: Emulated, cleared to 0.
7683	*/
7684	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7685	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7686	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7687	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7688	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7689	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7690	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7691	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7692	/** @todo NSTVMX: Save and adjust preemption timer value. */
7693	}
7694
7695	VMXLOCAL_BREAK_RC(rc);
7696	}
7697	# endif
7698	#endif
7699	}
7700	} while (0);
7701
7702	if (RT_SUCCESS(rc))
7703	{
7704	/* Update fExtrn. */
7705	pCtx->fExtrn &= ~fWhat;
7706
7707	/* If everything has been imported, clear the HM keeper bit. */
7708	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7709	{
7710	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7711	Assert(!pCtx->fExtrn);
7712	}
7713	}
7714	}
7715	else
7716	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7717
7718	ASMSetFlags(fEFlags);
7719
7720	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7721
7722	if (RT_SUCCESS(rc))
7723	{ /* likely */ }
7724	else
7725	return rc;
7726
7727	/*
7728	* Honor any pending CR3 updates.
7729	*
7730	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7731	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7732	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7733	*
7734	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7735	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7736	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7737	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7738	*
7739	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7740	*/
7741	if (VMMRZCallRing3IsEnabled(pVCpu))
7742	{
7743	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7744	{
7745	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7746	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7747	}
7748
7749	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7750	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7751
7752	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7753	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7754	}
7755
7756	return VINF_SUCCESS;
7757	#undef VMXLOCAL_BREAK_RC
7758	}
7759
7760
7761	/**
7762	* Saves the guest state from the VMCS into the guest-CPU context.
7763	*
7764	* @returns VBox status code.
7765	* @param pVCpu The cross context virtual CPU structure.
7766	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7767	*/
7768	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7769	{
7770	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7771	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7772	}
7773
7774
7775	/**
7776	* Check per-VM and per-VCPU force flag actions that require us to go back to
7777	* ring-3 for one reason or another.
7778	*
7779	* @returns Strict VBox status code (i.e. informational status codes too)
7780	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7781	* ring-3.
7782	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7783	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7784	* interrupts)
7785	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7786	* all EMTs to be in ring-3.
7787	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7788	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7789	* to the EM loop.
7790	*
7791	* @param pVCpu The cross context virtual CPU structure.
7792	* @param fStepping Whether we are single-stepping the guest using the
7793	* hypervisor debugger.
7794	*/
7795	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7796	{
7797	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7798
7799	/*
7800	* Update pending interrupts into the APIC's IRR.
7801	*/
7802	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7803	APICUpdatePendingInterrupts(pVCpu);
7804
7805	/*
7806	* Anything pending? Should be more likely than not if we're doing a good job.
7807	*/
7808	PVM pVM = pVCpu->CTX_SUFF(pVM);
7809	if ( !fStepping
7810	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7811	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7812	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7813	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7814	return VINF_SUCCESS;
7815
7816	/* Pending PGM C3 sync. */
7817	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7818	{
7819	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7820	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7821	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7822	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7823	if (rcStrict2 != VINF_SUCCESS)
7824	{
7825	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7826	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7827	return rcStrict2;
7828	}
7829	}
7830
7831	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7832	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7833	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7834	{
7835	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7836	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7837	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7838	return rc2;
7839	}
7840
7841	/* Pending VM request packets, such as hardware interrupts. */
7842	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7843	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7844	{
7845	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7846	return VINF_EM_PENDING_REQUEST;
7847	}
7848
7849	/* Pending PGM pool flushes. */
7850	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7851	{
7852	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7853	return VINF_PGM_POOL_FLUSH_PENDING;
7854	}
7855
7856	/* Pending DMA requests. */
7857	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7858	{
7859	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7860	return VINF_EM_RAW_TO_R3;
7861	}
7862
7863	return VINF_SUCCESS;
7864	}
7865
7866
7867	/**
7868	* Converts any TRPM trap into a pending HM event. This is typically used when
7869	* entering from ring-3 (not longjmp returns).
7870	*
7871	* @param pVCpu The cross context virtual CPU structure.
7872	*/
7873	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7874	{
7875	Assert(TRPMHasTrap(pVCpu));
7876	Assert(!pVCpu->hm.s.Event.fPending);
7877
7878	uint8_t uVector;
7879	TRPMEVENT enmTrpmEvent;
7880	RTGCUINT uErrCode;
7881	RTGCUINTPTR GCPtrFaultAddress;
7882	uint8_t cbInstr;
7883
7884	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7885	AssertRC(rc);
7886
7887	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7888	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7889	if (enmTrpmEvent == TRPM_TRAP)
7890	{
7891	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7892	* generated using INT1 (ICEBP). */
7893	switch (uVector)
7894	{
7895	case X86_XCPT_NMI:
7896	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7897	break;
7898
7899	case X86_XCPT_BP:
7900	case X86_XCPT_OF:
7901	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7902	break;
7903
7904	case X86_XCPT_PF:
7905	case X86_XCPT_DF:
7906	case X86_XCPT_TS:
7907	case X86_XCPT_NP:
7908	case X86_XCPT_SS:
7909	case X86_XCPT_GP:
7910	case X86_XCPT_AC:
7911	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7912	RT_FALL_THRU();
7913	default:
7914	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7915	break;
7916	}
7917	}
7918	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7919	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7920	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7921	{
7922	switch (uVector)
7923	{
7924	case X86_XCPT_BP:
7925	case X86_XCPT_OF:
7926	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7927	break;
7928
7929	default:
7930	Assert(uVector == X86_XCPT_DB);
7931	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7932	break;
7933	}
7934	}
7935	else
7936	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7937
7938	rc = TRPMResetTrap(pVCpu);
7939	AssertRC(rc);
7940	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7941	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7942
7943	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7944	}
7945
7946
7947	/**
7948	* Converts the pending HM event into a TRPM trap.
7949	*
7950	* @param pVCpu The cross context virtual CPU structure.
7951	*/
7952	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7953	{
7954	Assert(pVCpu->hm.s.Event.fPending);
7955
7956	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7957	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7958	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7959	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7960
7961	/* If a trap was already pending, we did something wrong! */
7962	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7963
7964	/** @todo Use HMVmxEventToTrpmEventType() later. */
7965	TRPMEVENT enmTrapType;
7966	switch (uVectorType)
7967	{
7968	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7969	enmTrapType = TRPM_HARDWARE_INT;
7970	break;
7971
7972	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7973	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7974	enmTrapType = TRPM_TRAP;
7975	break;
7976
7977	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7978	Assert(uVector == X86_XCPT_DB);
7979	enmTrapType = TRPM_SOFTWARE_INT;
7980	break;
7981
7982	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7983	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7984	enmTrapType = TRPM_SOFTWARE_INT;
7985	break;
7986
7987	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7988	enmTrapType = TRPM_SOFTWARE_INT;
7989	break;
7990
7991	default:
7992	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7993	enmTrapType = TRPM_32BIT_HACK;
7994	break;
7995	}
7996
7997	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7998
7999	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8000	AssertRC(rc);
8001
8002	if (fErrorCodeValid)
8003	TRPMSetErrorCode(pVCpu, uErrorCode);
8004
8005	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8006	&& uVector == X86_XCPT_PF)
8007	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8008	else if (enmTrapType == TRPM_SOFTWARE_INT)
8009	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8010
8011	/* We're now done converting the pending event. */
8012	pVCpu->hm.s.Event.fPending = false;
8013	}
8014
8015
8016	/**
8017	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8018	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8019	*
8020	* @param pVCpu The cross context virtual CPU structure.
8021	* @param pVmcsInfo The VMCS info. object.
8022	*/
8023	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8024	{
8025	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8026	{
8027	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8028	{
8029	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8030	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8031	AssertRC(rc);
8032	}
8033	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8034	}
8035
8036
8037	/**
8038	* Clears the interrupt-window exiting control in the VMCS.
8039	*
8040	* @param pVmcsInfo The VMCS info. object.
8041	*/
8042	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8043	{
8044	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8045	{
8046	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8047	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8048	}
8049	return VINF_SUCCESS;
8050	}
8051
8052
8053	/**
8054	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8055	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8056	*
8057	* @param pVCpu The cross context virtual CPU structure.
8058	* @param pVmcsInfo The VMCS info. object.
8059	*/
8060	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8061	{
8062	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8063	{
8064	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8065	{
8066	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8067	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8068	AssertRC(rc);
8069	Log4Func(("Setup NMI-window exiting\n"));
8070	}
8071	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8072	}
8073
8074
8075	/**
8076	* Clears the NMI-window exiting control in the VMCS.
8077	*
8078	* @param pVmcsInfo The VMCS info. object.
8079	*/
8080	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8081	{
8082	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8083	{
8084	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8085	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8086	}
8087	return VINF_SUCCESS;
8088	}
8089
8090
8091	/**
8092	* Does the necessary state syncing before returning to ring-3 for any reason
8093	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8094	*
8095	* @returns VBox status code.
8096	* @param pVCpu The cross context virtual CPU structure.
8097	* @param fImportState Whether to import the guest state from the VMCS back
8098	* to the guest-CPU context.
8099	*
8100	* @remarks No-long-jmp zone!!!
8101	*/
8102	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8103	{
8104	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8105	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8106
8107	RTCPUID idCpu = RTMpCpuId();
8108	Log4Func(("HostCpuId=%u\n", idCpu));
8109
8110	/*
8111	* !!! IMPORTANT !!!
8112	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8113	*/
8114
8115	/* Save the guest state if necessary. */
8116	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8117	if (fImportState)
8118	{
8119	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8120	AssertRCReturn(rc, rc);
8121	}
8122
8123	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8124	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8125	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8126
8127	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8128	#ifdef VBOX_STRICT
8129	if (CPUMIsHyperDebugStateActive(pVCpu))
8130	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8131	#endif
8132	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8133	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8134	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8135
8136	#if HC_ARCH_BITS == 64
8137	/* Restore host-state bits that VT-x only restores partially. */
8138	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8139	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8140	{
8141	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8142	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8143	}
8144	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8145	#endif
8146
8147	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8148	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8149	{
8150	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8151	if (!fImportState)
8152	{
8153	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8154	AssertRCReturn(rc, rc);
8155	}
8156	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8157	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8158	}
8159	else
8160	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8161
8162	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8163	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8164
8165	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8166	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8167	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8168	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8169	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8170	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8171	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8172	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8173	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8174
8175	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8176
8177	/** @todo This partially defeats the purpose of having preemption hooks.
8178	* The problem is, deregistering the hooks should be moved to a place that
8179	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8180	* context.
8181	*/
8182	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8183	AssertRCReturn(rc, rc);
8184
8185	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8186	NOREF(idCpu);
8187	return VINF_SUCCESS;
8188	}
8189
8190
8191	/**
8192	* Leaves the VT-x session.
8193	*
8194	* @returns VBox status code.
8195	* @param pVCpu The cross context virtual CPU structure.
8196	*
8197	* @remarks No-long-jmp zone!!!
8198	*/
8199	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8200	{
8201	HM_DISABLE_PREEMPT(pVCpu);
8202	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8203	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8204	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8205
8206	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8207	and done this from the VMXR0ThreadCtxCallback(). */
8208	if (!pVCpu->hm.s.fLeaveDone)
8209	{
8210	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8211	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8212	pVCpu->hm.s.fLeaveDone = true;
8213	}
8214	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8215
8216	/*
8217	* !!! IMPORTANT !!!
8218	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8219	*/
8220
8221	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8222	/** @todo Deregistering here means we need to VMCLEAR always
8223	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8224	* for calling VMMR0ThreadCtxHookDisable here! */
8225	VMMR0ThreadCtxHookDisable(pVCpu);
8226
8227	/* Leave HM context. This takes care of local init (term). */
8228	int rc = HMR0LeaveCpu(pVCpu);
8229
8230	HM_RESTORE_PREEMPT();
8231	return rc;
8232	}
8233
8234
8235	/**
8236	* Does the necessary state syncing before doing a longjmp to ring-3.
8237	*
8238	* @returns VBox status code.
8239	* @param pVCpu The cross context virtual CPU structure.
8240	*
8241	* @remarks No-long-jmp zone!!!
8242	*/
8243	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8244	{
8245	return hmR0VmxLeaveSession(pVCpu);
8246	}
8247
8248
8249	/**
8250	* Take necessary actions before going back to ring-3.
8251	*
8252	* An action requires us to go back to ring-3. This function does the necessary
8253	* steps before we can safely return to ring-3. This is not the same as longjmps
8254	* to ring-3, this is voluntary and prepares the guest so it may continue
8255	* executing outside HM (recompiler/IEM).
8256	*
8257	* @returns VBox status code.
8258	* @param pVCpu The cross context virtual CPU structure.
8259	* @param rcExit The reason for exiting to ring-3. Can be
8260	* VINF_VMM_UNKNOWN_RING3_CALL.
8261	*/
8262	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8263	{
8264	Assert(pVCpu);
8265	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8266
8267	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8268	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8269	{
8270	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8271	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8272	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8273	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8274	}
8275
8276	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8277	VMMRZCallRing3Disable(pVCpu);
8278	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8279
8280	/*
8281	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8282	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8283	*
8284	* This is because execution may continue from ring-3 and we would need to inject
8285	* the event from there (hence place it back in TRPM).
8286	*/
8287	if (pVCpu->hm.s.Event.fPending)
8288	{
8289	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8290	Assert(!pVCpu->hm.s.Event.fPending);
8291
8292	/* Clear the events from the VMCS. */
8293	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8294	AssertRCReturn(rc, rc);
8295	}
8296	#ifdef VBOX_STRICT
8297	else
8298	{
8299	/*
8300	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8301	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8302	* occasionally, see @bugref{9180#c42}.
8303	*/
8304	uint32_t uEntryIntInfo;
8305	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8306	AssertRC(rc);
8307	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8308	}
8309	#endif
8310
8311	/*
8312	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8313	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8314	* (e.g. TPR below threshold).
8315	*/
8316	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8317	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8318	AssertRCReturn(rc, rc);
8319
8320	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8321	and if we're injecting an event we should have a TRPM trap pending. */
8322	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8323	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8324	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8325	#endif
8326
8327	/* Save guest state and restore host state bits. */
8328	rc = hmR0VmxLeaveSession(pVCpu);
8329	AssertRCReturn(rc, rc);
8330	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8331
8332	/* Thread-context hooks are unregistered at this point!!! */
8333
8334	/* Sync recompiler state. */
8335	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8336	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8337	\| CPUM_CHANGED_LDTR
8338	\| CPUM_CHANGED_GDTR
8339	\| CPUM_CHANGED_IDTR
8340	\| CPUM_CHANGED_TR
8341	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8342	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8343	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8344	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8345
8346	Assert(!pVCpu->hm.s.fClearTrapFlag);
8347
8348	/* Update the exit-to-ring 3 reason. */
8349	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8350
8351	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8352	if ( rcExit != VINF_EM_RAW_INTERRUPT
8353	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8354	{
8355	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8356	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8357	}
8358
8359	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8360
8361	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8362	VMMRZCallRing3RemoveNotification(pVCpu);
8363	VMMRZCallRing3Enable(pVCpu);
8364
8365	return rc;
8366	}
8367
8368
8369	/**
8370	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8371	* longjump to ring-3 and possibly get preempted.
8372	*
8373	* @returns VBox status code.
8374	* @param pVCpu The cross context virtual CPU structure.
8375	* @param enmOperation The operation causing the ring-3 longjump.
8376	* @param pvUser User argument, currently unused, NULL.
8377	*/
8378	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8379	{
8380	RT_NOREF(pvUser);
8381	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8382	{
8383	/*
8384	* !!! IMPORTANT !!!
8385	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8386	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8387	*/
8388	VMMRZCallRing3RemoveNotification(pVCpu);
8389	VMMRZCallRing3Disable(pVCpu);
8390	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8391	RTThreadPreemptDisable(&PreemptState);
8392
8393	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8394	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8395	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8396	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8397
8398	#if HC_ARCH_BITS == 64
8399	/* Restore host-state bits that VT-x only restores partially. */
8400	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8401	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8402	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8403	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8404	#endif
8405
8406	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8407	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8408	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8409
8410	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8411	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8412	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8413
8414	/* Clear the current VMCS data back to memory. */
8415	hmR0VmxClearVmcs(pVmcsInfo);
8416
8417	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8418	VMMR0ThreadCtxHookDisable(pVCpu);
8419	HMR0LeaveCpu(pVCpu);
8420	RTThreadPreemptRestore(&PreemptState);
8421	return VINF_SUCCESS;
8422	}
8423
8424	Assert(pVCpu);
8425	Assert(pvUser);
8426	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8427	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8428
8429	VMMRZCallRing3Disable(pVCpu);
8430	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8431
8432	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8433
8434	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8435	AssertRCReturn(rc, rc);
8436
8437	VMMRZCallRing3Enable(pVCpu);
8438	return VINF_SUCCESS;
8439	}
8440
8441
8442	/**
8443	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8444	* stack.
8445	*
8446	* @returns Strict VBox status code (i.e. informational status codes too).
8447	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8448	* @param pVCpu The cross context virtual CPU structure.
8449	* @param uValue The value to push to the guest stack.
8450	*/
8451	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8452	{
8453	/*
8454	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8455	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8456	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8457	*/
8458	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8459	if (pCtx->sp == 1)
8460	return VINF_EM_RESET;
8461	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8462	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8463	AssertRC(rc);
8464	return rc;
8465	}
8466
8467
8468	/**
8469	* Injects an event into the guest upon VM-entry by updating the relevant fields
8470	* in the VM-entry area in the VMCS.
8471	*
8472	* @returns Strict VBox status code (i.e. informational status codes too).
8473	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8474	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8475	*
8476	* @param pVCpu The cross context virtual CPU structure.
8477	* @param pVmxTransient The VMX-transient structure.
8478	* @param pEvent The event being injected.
8479	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8480	* This will be updated if necessary. This cannot not
8481	* be NULL.
8482	* @param fStepping Whether we're single-stepping guest execution and
8483	* should return VINF_EM_DBG_STEPPED if the event is
8484	* injected directly (registers modified by us, not by
8485	* hardware on VM-entry).
8486	*/
8487	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8488	uint32_t *pfIntrState)
8489	{
8490	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8491	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8492	Assert(pfIntrState);
8493
8494	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8495	uint32_t u32IntInfo = pEvent->u64IntInfo;
8496	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8497	uint32_t const cbInstr = pEvent->cbInstr;
8498	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8499	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8500	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8501
8502	#ifdef VBOX_STRICT
8503	/*
8504	* Validate the error-code-valid bit for hardware exceptions.
8505	* No error codes for exceptions in real-mode.
8506	*
8507	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8508	*/
8509	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8510	&& !CPUMIsGuestInRealModeEx(pCtx))
8511	{
8512	switch (uVector)
8513	{
8514	case X86_XCPT_PF:
8515	case X86_XCPT_DF:
8516	case X86_XCPT_TS:
8517	case X86_XCPT_NP:
8518	case X86_XCPT_SS:
8519	case X86_XCPT_GP:
8520	case X86_XCPT_AC:
8521	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8522	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8523	RT_FALL_THRU();
8524	default:
8525	break;
8526	}
8527	}
8528
8529	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8530	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8531	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8532	#endif
8533
8534	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8535
8536	/*
8537	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8538	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8539	* interrupt handler in the (real-mode) guest.
8540	*
8541	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8542	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8543	*/
8544	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8545	{
8546	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8547	{
8548	/*
8549	* For CPUs with unrestricted guest execution enabled and with the guest
8550	* in real-mode, we must not set the deliver-error-code bit.
8551	*
8552	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8553	*/
8554	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8555	}
8556	else
8557	{
8558	PVM pVM = pVCpu->CTX_SUFF(pVM);
8559	Assert(PDMVmmDevHeapIsEnabled(pVM));
8560	Assert(pVM->hm.s.vmx.pRealModeTSS);
8561	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8562
8563	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8564	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8565	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8566	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8567	AssertRCReturn(rc2, rc2);
8568
8569	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8570	size_t const cbIdtEntry = sizeof(X86IDTR16);
8571	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8572	{
8573	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8574	if (uVector == X86_XCPT_DF)
8575	return VINF_EM_RESET;
8576
8577	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8578	No error codes for exceptions in real-mode. */
8579	if (uVector == X86_XCPT_GP)
8580	{
8581	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8582	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8583	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8584	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8585	HMEVENT EventXcptDf;
8586	RT_ZERO(EventXcptDf);
8587	EventXcptDf.u64IntInfo = uXcptDfInfo;
8588	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8589	}
8590
8591	/*
8592	* If we're injecting an event with no valid IDT entry, inject a #GP.
8593	* No error codes for exceptions in real-mode.
8594	*
8595	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8596	*/
8597	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8598	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8599	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8600	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8601	HMEVENT EventXcptGp;
8602	RT_ZERO(EventXcptGp);
8603	EventXcptGp.u64IntInfo = uXcptGpInfo;
8604	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8605	}
8606
8607	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8608	uint16_t uGuestIp = pCtx->ip;
8609	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8610	{
8611	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8612	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8613	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8614	}
8615	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8616	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8617
8618	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8619	X86IDTR16 IdtEntry;
8620	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8621	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8622	AssertRCReturn(rc2, rc2);
8623
8624	/* Construct the stack frame for the interrupt/exception handler. */
8625	VBOXSTRICTRC rcStrict;
8626	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8627	if (rcStrict == VINF_SUCCESS)
8628	{
8629	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8630	if (rcStrict == VINF_SUCCESS)
8631	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8632	}
8633
8634	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8635	if (rcStrict == VINF_SUCCESS)
8636	{
8637	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8638	pCtx->rip = IdtEntry.offSel;
8639	pCtx->cs.Sel = IdtEntry.uSel;
8640	pCtx->cs.ValidSel = IdtEntry.uSel;
8641	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8642	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8643	&& uVector == X86_XCPT_PF)
8644	pCtx->cr2 = GCPtrFault;
8645
8646	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8647	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8648	\| HM_CHANGED_GUEST_RSP);
8649
8650	/*
8651	* If we delivered a hardware exception (other than an NMI) and if there was
8652	* block-by-STI in effect, we should clear it.
8653	*/
8654	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8655	{
8656	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8657	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8658	Log4Func(("Clearing inhibition due to STI\n"));
8659	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8660	}
8661
8662	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8663	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8664
8665	/*
8666	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8667	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8668	*/
8669	pVCpu->hm.s.Event.fPending = false;
8670
8671	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8672	if (fStepping)
8673	rcStrict = VINF_EM_DBG_STEPPED;
8674	}
8675	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8676	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8677	return rcStrict;
8678	}
8679	}
8680
8681	/*
8682	* Validate.
8683	*/
8684	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8685	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8686
8687	/*
8688	* Inject the event into the VMCS.
8689	*/
8690	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8691	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8692	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8693	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8694	AssertRCReturn(rc, rc);
8695
8696	/*
8697	* Update guest CR2 if this is a page-fault.
8698	*/
8699	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8700	&& uVector == X86_XCPT_PF)
8701	pCtx->cr2 = GCPtrFault;
8702
8703	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8704	return VINF_SUCCESS;
8705	}
8706
8707
8708	/**
8709	* Evaluates the event to be delivered to the guest and sets it as the pending
8710	* event.
8711	*
8712	* @returns Strict VBox status code (i.e. informational status codes too).
8713	* @param pVCpu The cross context virtual CPU structure.
8714	* @param pVmxTransient The VMX-transient structure.
8715	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8716	*/
8717	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8718	{
8719	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8720	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8721
8722	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8723	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8724	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8725	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8726	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8727
8728	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8729	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8730	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8731	Assert(!TRPMHasTrap(pVCpu));
8732	Assert(pfIntrState);
8733
8734	*pfIntrState = fIntrState;
8735
8736	/*
8737	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8738	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8739	*/
8740	/** @todo SMI. SMIs take priority over NMIs. */
8741	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8742	{
8743	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8744	if ( !pVCpu->hm.s.Event.fPending
8745	&& !fBlockNmi
8746	&& !fBlockSti
8747	&& !fBlockMovSS)
8748	{
8749	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8750	if ( pVmxTransient->fIsNestedGuest
8751	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8752	return IEMExecVmxVmexitNmi(pVCpu);
8753	#endif
8754	hmR0VmxSetPendingXcptNmi(pVCpu);
8755	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8756	Log4Func(("Pending NMI\n"));
8757	}
8758	else
8759	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8760	}
8761	/*
8762	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8763	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8764	*/
8765	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8766	&& !pVCpu->hm.s.fSingleInstruction)
8767	{
8768	Assert(!DBGFIsStepping(pVCpu));
8769	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8770	AssertRCReturn(rc, rc);
8771	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8772	if ( !pVCpu->hm.s.Event.fPending
8773	&& !fBlockInt
8774	&& !fBlockSti
8775	&& !fBlockMovSS)
8776	{
8777	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8778	if ( pVmxTransient->fIsNestedGuest
8779	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8780	{
8781	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8782	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8783	return rcStrict;
8784	}
8785	#endif
8786	uint8_t u8Interrupt;
8787	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8788	if (RT_SUCCESS(rc))
8789	{
8790	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8791	if ( pVmxTransient->fIsNestedGuest
8792	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8793	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8794	{
8795	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8796	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8797	return rcStrict;
8798	}
8799	#endif
8800	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8801	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8802	}
8803	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8804	{
8805	if ( !pVmxTransient->fIsNestedGuest
8806	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8807	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8808	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8809
8810	/*
8811	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8812	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8813	* need to re-set this force-flag here.
8814	*/
8815	}
8816	else
8817	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8818	}
8819	else
8820	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8821	}
8822
8823	return VINF_SUCCESS;
8824	}
8825
8826
8827	/**
8828	* Injects any pending events into the guest if the guest is in a state to
8829	* receive them.
8830	*
8831	* @returns Strict VBox status code (i.e. informational status codes too).
8832	* @param pVCpu The cross context virtual CPU structure.
8833	* @param pVmxTransient The VMX-transient structure.
8834	* @param fIntrState The VT-x guest-interruptibility state.
8835	* @param fStepping Whether we are single-stepping the guest using the
8836	* hypervisor debugger and should return
8837	* VINF_EM_DBG_STEPPED if the event was dispatched
8838	* directly.
8839	*/
8840	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8841	{
8842	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8843	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8844
8845	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8846	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8847
8848	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8849	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8850	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8851	Assert(!TRPMHasTrap(pVCpu));
8852
8853	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8854	if (pVCpu->hm.s.Event.fPending)
8855	{
8856	/*
8857	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8858	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8859	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8860	*
8861	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8862	*/
8863	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8864	#ifdef VBOX_STRICT
8865	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8866	{
8867	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8868	Assert(!fBlockInt);
8869	Assert(!fBlockSti);
8870	Assert(!fBlockMovSS);
8871	}
8872	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8873	{
8874	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8875	Assert(!fBlockSti);
8876	Assert(!fBlockMovSS);
8877	Assert(!fBlockNmi);
8878	}
8879	#endif
8880	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8881	uIntType));
8882
8883	/*
8884	* Inject the event and get any changes to the guest-interruptibility state.
8885	*
8886	* The guest-interruptibility state may need to be updated if we inject the event
8887	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8888	*/
8889	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8890	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8891
8892	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8893	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8894	else
8895	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8896	}
8897
8898	/*
8899	* Update the guest-interruptibility state.
8900	*
8901	* This is required for the real-on-v86 software interrupt injection case above, as well as
8902	* updates to the guest state from ring-3 or IEM/REM.
8903	*/
8904	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8905	AssertRCReturn(rc, rc);
8906
8907	/*
8908	* There's no need to clear the VM-entry interruption-information field here if we're not
8909	* injecting anything. VT-x clears the valid bit on every VM-exit.
8910	*
8911	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8912	*/
8913
8914	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8915	NOREF(fBlockMovSS); NOREF(fBlockSti);
8916	return rcStrict;
8917	}
8918
8919
8920	/**
8921	* Enters the VT-x session.
8922	*
8923	* @returns VBox status code.
8924	* @param pVCpu The cross context virtual CPU structure.
8925	*/
8926	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8927	{
8928	AssertPtr(pVCpu);
8929	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8930	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8931
8932	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8933	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8934	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8935
8936	#ifdef VBOX_STRICT
8937	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8938	RTCCUINTREG uHostCR4 = ASMGetCR4();
8939	if (!(uHostCR4 & X86_CR4_VMXE))
8940	{
8941	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8942	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8943	}
8944	#endif
8945
8946	/*
8947	* Load the appropriate VMCS as the current and active one.
8948	*/
8949	PVMXVMCSINFO pVmcsInfo;
8950	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8951	if (!fInNestedGuestMode)
8952	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8953	else
8954	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8955	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8956	if (RT_SUCCESS(rc))
8957	{
8958	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8959	pVCpu->hm.s.fLeaveDone = false;
8960	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8961
8962	/*
8963	* Do the EMT scheduled L1D flush here if needed.
8964	*/
8965	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8966	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8967	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
8968	hmR0MdsClear();
8969	}
8970	return rc;
8971	}
8972
8973
8974	/**
8975	* The thread-context callback (only on platforms which support it).
8976	*
8977	* @param enmEvent The thread-context event.
8978	* @param pVCpu The cross context virtual CPU structure.
8979	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8980	* @thread EMT(pVCpu)
8981	*/
8982	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8983	{
8984	NOREF(fGlobalInit);
8985
8986	switch (enmEvent)
8987	{
8988	case RTTHREADCTXEVENT_OUT:
8989	{
8990	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8991	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8992	VMCPU_ASSERT_EMT(pVCpu);
8993
8994	/* No longjmps (logger flushes, locks) in this fragile context. */
8995	VMMRZCallRing3Disable(pVCpu);
8996	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8997
8998	/* Restore host-state (FPU, debug etc.) */
8999	if (!pVCpu->hm.s.fLeaveDone)
9000	{
9001	/*
9002	* Do -not- import the guest-state here as we might already be in the middle of importing
9003	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9004	*/
9005	hmR0VmxLeave(pVCpu, false /* fImportState */);
9006	pVCpu->hm.s.fLeaveDone = true;
9007	}
9008
9009	/* Leave HM context, takes care of local init (term). */
9010	int rc = HMR0LeaveCpu(pVCpu);
9011	AssertRC(rc);
9012
9013	/* Restore longjmp state. */
9014	VMMRZCallRing3Enable(pVCpu);
9015	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9016	break;
9017	}
9018
9019	case RTTHREADCTXEVENT_IN:
9020	{
9021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9022	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9023	VMCPU_ASSERT_EMT(pVCpu);
9024
9025	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9026	VMMRZCallRing3Disable(pVCpu);
9027	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9028
9029	/* Initialize the bare minimum state required for HM. This takes care of
9030	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9031	int rc = hmR0EnterCpu(pVCpu);
9032	AssertRC(rc);
9033	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9034	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9035
9036	/* Load the active VMCS as the current one. */
9037	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9038	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9039	AssertRC(rc);
9040	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9041	pVCpu->hm.s.fLeaveDone = false;
9042
9043	/* Do the EMT scheduled L1D flush if needed. */
9044	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9045	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9046
9047	/* Restore longjmp state. */
9048	VMMRZCallRing3Enable(pVCpu);
9049	break;
9050	}
9051
9052	default:
9053	break;
9054	}
9055	}
9056
9057
9058	/**
9059	* Exports the host state into the VMCS host-state area.
9060	* Sets up the VM-exit MSR-load area.
9061	*
9062	* The CPU state will be loaded from these fields on every successful VM-exit.
9063	*
9064	* @returns VBox status code.
9065	* @param pVCpu The cross context virtual CPU structure.
9066	*
9067	* @remarks No-long-jump zone!!!
9068	*/
9069	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9070	{
9071	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9072
9073	int rc = VINF_SUCCESS;
9074	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9075	{
9076	rc = hmR0VmxExportHostControlRegs();
9077	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9078
9079	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9080	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9081
9082	rc = hmR0VmxExportHostMsrs(pVCpu);
9083	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9084
9085	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9086	}
9087	return rc;
9088	}
9089
9090
9091	/**
9092	* Saves the host state in the VMCS host-state.
9093	*
9094	* @returns VBox status code.
9095	* @param pVCpu The cross context virtual CPU structure.
9096	*
9097	* @remarks No-long-jump zone!!!
9098	*/
9099	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9100	{
9101	AssertPtr(pVCpu);
9102	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9103
9104	/*
9105	* Export the host state here while entering HM context.
9106	* When thread-context hooks are used, we might get preempted and have to re-save the host
9107	* state but most of the time we won't be, so do it here before we disable interrupts.
9108	*/
9109	return hmR0VmxExportHostState(pVCpu);
9110	}
9111
9112
9113	/**
9114	* Exports the guest state into the VMCS guest-state area.
9115	*
9116	* The will typically be done before VM-entry when the guest-CPU state and the
9117	* VMCS state may potentially be out of sync.
9118	*
9119	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9120	* VM-entry controls.
9121	* Sets up the appropriate VMX non-root function to execute guest code based on
9122	* the guest CPU mode.
9123	*
9124	* @returns VBox strict status code.
9125	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9126	* without unrestricted guest execution and the VMMDev is not presently
9127	* mapped (e.g. EFI32).
9128	*
9129	* @param pVCpu The cross context virtual CPU structure.
9130	* @param pVmxTransient The VMX-transient structure.
9131	*
9132	* @remarks No-long-jump zone!!!
9133	*/
9134	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9135	{
9136	AssertPtr(pVCpu);
9137	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9138	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9139
9140	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9141
9142	/*
9143	* Determine real-on-v86 mode.
9144	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9145	*/
9146	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9147	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9148	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9149	pVmcsInfo->RealMode. fRealOnV86Active = false;
9150	else
9151	{
9152	Assert(!pVmxTransient->fIsNestedGuest);
9153	pVmcsInfo->RealMode.fRealOnV86Active = true;
9154	}
9155
9156	/*
9157	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9158	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9159	*/
9160	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9161	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9162	* be a need to evaluate this everytime since I'm pretty sure we intercept
9163	* all guest paging mode changes. */
9164	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9165	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9166
9167	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9168	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9169
9170	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9171	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9172
9173	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9174	if (rcStrict == VINF_SUCCESS)
9175	{ /* likely */ }
9176	else
9177	{
9178	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9179	return rcStrict;
9180	}
9181
9182	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9183	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9184
9185	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9186	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9187
9188	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9189	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9190
9191	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9192	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9193
9194	rc = hmR0VmxExportGuestRip(pVCpu);
9195	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9196	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9197	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9198
9199	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9200	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9201	\| HM_CHANGED_GUEST_CR2
9202	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9203	\| HM_CHANGED_GUEST_X87
9204	\| HM_CHANGED_GUEST_SSE_AVX
9205	\| HM_CHANGED_GUEST_OTHER_XSAVE
9206	\| HM_CHANGED_GUEST_XCRx
9207	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9208	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9209	\| HM_CHANGED_GUEST_TSC_AUX
9210	\| HM_CHANGED_GUEST_OTHER_MSRS
9211	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9212	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9213
9214	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9215	return rc;
9216	}
9217
9218
9219	/**
9220	* Exports the state shared between the host and guest into the VMCS.
9221	*
9222	* @param pVCpu The cross context virtual CPU structure.
9223	* @param pVmxTransient The VMX-transient structure.
9224	*
9225	* @remarks No-long-jump zone!!!
9226	*/
9227	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9228	{
9229	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9230	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9231
9232	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9233	{
9234	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9235	AssertRC(rc);
9236	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9237
9238	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9239	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9240	{
9241	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9242	AssertRC(rc);
9243	}
9244	}
9245
9246	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9247	{
9248	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9249	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9250	}
9251
9252	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9253	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9254	}
9255
9256
9257	/**
9258	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9259	*
9260	* @returns Strict VBox status code (i.e. informational status codes too).
9261	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9262	* without unrestricted guest execution and the VMMDev is not presently
9263	* mapped (e.g. EFI32).
9264	*
9265	* @param pVCpu The cross context virtual CPU structure.
9266	* @param pVmxTransient The VMX-transient structure.
9267	*
9268	* @remarks No-long-jump zone!!!
9269	*/
9270	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9271	{
9272	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9273	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9274	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9275
9276	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9278	#endif
9279
9280	/*
9281	* For many exits it's only RIP that changes and hence try to export it first
9282	* without going through a lot of change flag checks.
9283	*/
9284	VBOXSTRICTRC rcStrict;
9285	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9286	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9287	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9288	{
9289	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9290	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9291	{ /* likely */}
9292	else
9293	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9295	}
9296	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9297	{
9298	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9299	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9300	{ /* likely */}
9301	else
9302	{
9303	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9304	VBOXSTRICTRC_VAL(rcStrict)));
9305	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9306	return rcStrict;
9307	}
9308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9309	}
9310	else
9311	rcStrict = VINF_SUCCESS;
9312
9313	#ifdef VBOX_STRICT
9314	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9315	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9316	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9317	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9318	("fCtxChanged=%#RX64\n", fCtxChanged));
9319	#endif
9320	return rcStrict;
9321	}
9322
9323
9324	/**
9325	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9326	* and update error record fields accordingly.
9327	*
9328	* @return VMX_IGS_* return codes.
9329	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9330	* wrong with the guest state.
9331	*
9332	* @param pVCpu The cross context virtual CPU structure.
9333	* @param pVmcsInfo The VMCS info. object.
9334	*
9335	* @remarks This function assumes our cache of the VMCS controls
9336	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9337	*/
9338	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9339	{
9340	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9341	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9342	uError = (err); \
9343	break; \
9344	} else do { } while (0)
9345
9346	int rc;
9347	PVM pVM = pVCpu->CTX_SUFF(pVM);
9348	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9349	uint32_t uError = VMX_IGS_ERROR;
9350	uint32_t u32Val;
9351	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9352
9353	do
9354	{
9355	/*
9356	* CR0.
9357	*/
9358	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9359	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9360	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9361	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9362	if (fUnrestrictedGuest)
9363	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9364
9365	uint32_t u32GuestCr0;
9366	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9367	AssertRCBreak(rc);
9368	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9369	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9370	if ( !fUnrestrictedGuest
9371	&& (u32GuestCr0 & X86_CR0_PG)
9372	&& !(u32GuestCr0 & X86_CR0_PE))
9373	{
9374	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9375	}
9376
9377	/*
9378	* CR4.
9379	*/
9380	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9381	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9382
9383	uint32_t u32GuestCr4;
9384	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9385	AssertRCBreak(rc);
9386	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9387	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9388
9389	/*
9390	* IA32_DEBUGCTL MSR.
9391	*/
9392	uint64_t u64Val;
9393	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9394	AssertRCBreak(rc);
9395	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9396	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9397	{
9398	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9399	}
9400	uint64_t u64DebugCtlMsr = u64Val;
9401
9402	#ifdef VBOX_STRICT
9403	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9404	AssertRCBreak(rc);
9405	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9406	#endif
9407	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9408
9409	/*
9410	* RIP and RFLAGS.
9411	*/
9412	uint32_t u32Eflags;
9413	#if HC_ARCH_BITS == 64
9414	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9415	AssertRCBreak(rc);
9416	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9417	if ( !fLongModeGuest
9418	\|\| !pCtx->cs.Attr.n.u1Long)
9419	{
9420	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9421	}
9422	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9423	* must be identical if the "IA-32e mode guest" VM-entry
9424	* control is 1 and CS.L is 1. No check applies if the
9425	* CPU supports 64 linear-address bits. */
9426
9427	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9428	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9429	AssertRCBreak(rc);
9430	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9431	VMX_IGS_RFLAGS_RESERVED);
9432	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9433	u32Eflags = u64Val;
9434	#else
9435	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9436	AssertRCBreak(rc);
9437	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9438	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9439	#endif
9440
9441	if ( fLongModeGuest
9442	\|\| ( fUnrestrictedGuest
9443	&& !(u32GuestCr0 & X86_CR0_PE)))
9444	{
9445	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9446	}
9447
9448	uint32_t u32EntryInfo;
9449	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9450	AssertRCBreak(rc);
9451	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9452	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9453	{
9454	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9455	}
9456
9457	/*
9458	* 64-bit checks.
9459	*/
9460	#if HC_ARCH_BITS == 64
9461	if (fLongModeGuest)
9462	{
9463	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9464	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9465	}
9466
9467	if ( !fLongModeGuest
9468	&& (u32GuestCr4 & X86_CR4_PCIDE))
9469	{
9470	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9471	}
9472
9473	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9474	* 51:32 beyond the processor's physical-address width are 0. */
9475
9476	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9477	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9478	{
9479	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9480	}
9481
9482	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9483	AssertRCBreak(rc);
9484	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9485
9486	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9487	AssertRCBreak(rc);
9488	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9489	#endif
9490
9491	/*
9492	* PERF_GLOBAL MSR.
9493	*/
9494	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9495	{
9496	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9497	AssertRCBreak(rc);
9498	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9499	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9500	}
9501
9502	/*
9503	* PAT MSR.
9504	*/
9505	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9506	{
9507	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9508	AssertRCBreak(rc);
9509	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9510	for (unsigned i = 0; i < 8; i++)
9511	{
9512	uint8_t u8Val = (u64Val & 0xff);
9513	if ( u8Val != 0 /* UC */
9514	&& u8Val != 1 /* WC */
9515	&& u8Val != 4 /* WT */
9516	&& u8Val != 5 /* WP */
9517	&& u8Val != 6 /* WB */
9518	&& u8Val != 7 /* UC- */)
9519	{
9520	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9521	}
9522	u64Val >>= 8;
9523	}
9524	}
9525
9526	/*
9527	* EFER MSR.
9528	*/
9529	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9530	{
9531	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9532	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9533	AssertRCBreak(rc);
9534	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9535	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9536	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9537	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9538	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9539	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9540	* iemVmxVmentryCheckGuestState(). */
9541	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9542	\|\| !(u32GuestCr0 & X86_CR0_PG)
9543	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9544	VMX_IGS_EFER_LMA_LME_MISMATCH);
9545	}
9546
9547	/*
9548	* Segment registers.
9549	*/
9550	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9551	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9552	if (!(u32Eflags & X86_EFL_VM))
9553	{
9554	/* CS */
9555	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9556	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9557	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9558	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9559	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9560	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9561	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9562	/* CS cannot be loaded with NULL in protected mode. */
9563	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9564	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9565	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9566	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9567	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9568	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9569	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9570	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9571	else
9572	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9573
9574	/* SS */
9575	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9576	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9577	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9578	if ( !(pCtx->cr0 & X86_CR0_PE)
9579	\|\| pCtx->cs.Attr.n.u4Type == 3)
9580	{
9581	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9582	}
9583	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9584	{
9585	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9586	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9587	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9588	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9589	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9590	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9591	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9592	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9593	}
9594
9595	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9596	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9597	{
9598	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9599	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9600	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9601	\|\| pCtx->ds.Attr.n.u4Type > 11
9602	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9603	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9604	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9605	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9606	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9607	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9608	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9609	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9610	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9611	}
9612	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9613	{
9614	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9615	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9616	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9617	\|\| pCtx->es.Attr.n.u4Type > 11
9618	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9619	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9620	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9621	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9622	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9623	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9624	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9625	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9626	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9627	}
9628	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9629	{
9630	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9631	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9632	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9633	\|\| pCtx->fs.Attr.n.u4Type > 11
9634	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9635	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9636	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9637	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9638	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9639	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9640	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9641	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9642	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9643	}
9644	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9645	{
9646	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9647	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9648	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9649	\|\| pCtx->gs.Attr.n.u4Type > 11
9650	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9651	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9652	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9653	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9654	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9655	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9656	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9657	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9658	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9659	}
9660	/* 64-bit capable CPUs. */
9661	#if HC_ARCH_BITS == 64
9662	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9663	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9664	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9665	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9666	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9667	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9668	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9669	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9670	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9671	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9672	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9673	#endif
9674	}
9675	else
9676	{
9677	/* V86 mode checks. */
9678	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9679	if (pVmcsInfo->RealMode.fRealOnV86Active)
9680	{
9681	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9682	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9683	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9684	}
9685	else
9686	{
9687	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9688	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9689	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9690	}
9691
9692	/* CS */
9693	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9694	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9695	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9696	/* SS */
9697	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9698	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9699	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9700	/* DS */
9701	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9702	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9703	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9704	/* ES */
9705	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9706	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9707	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9708	/* FS */
9709	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9710	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9711	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9712	/* GS */
9713	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9714	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9715	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9716	/* 64-bit capable CPUs. */
9717	#if HC_ARCH_BITS == 64
9718	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9719	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9720	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9721	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9722	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9723	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9724	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9725	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9726	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9727	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9728	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9729	#endif
9730	}
9731
9732	/*
9733	* TR.
9734	*/
9735	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9736	/* 64-bit capable CPUs. */
9737	#if HC_ARCH_BITS == 64
9738	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9739	#endif
9740	if (fLongModeGuest)
9741	{
9742	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9743	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9744	}
9745	else
9746	{
9747	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9748	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9749	VMX_IGS_TR_ATTR_TYPE_INVALID);
9750	}
9751	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9752	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9753	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9754	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9755	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9756	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9757	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9758	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9759
9760	/*
9761	* GDTR and IDTR.
9762	*/
9763	#if HC_ARCH_BITS == 64
9764	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9765	AssertRCBreak(rc);
9766	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9767
9768	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9769	AssertRCBreak(rc);
9770	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9771	#endif
9772
9773	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9774	AssertRCBreak(rc);
9775	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9776
9777	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9778	AssertRCBreak(rc);
9779	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9780
9781	/*
9782	* Guest Non-Register State.
9783	*/
9784	/* Activity State. */
9785	uint32_t u32ActivityState;
9786	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9787	AssertRCBreak(rc);
9788	HMVMX_CHECK_BREAK( !u32ActivityState
9789	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9790	VMX_IGS_ACTIVITY_STATE_INVALID);
9791	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9792	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9793	uint32_t u32IntrState;
9794	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9795	AssertRCBreak(rc);
9796	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9797	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9798	{
9799	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9800	}
9801
9802	/** @todo Activity state and injecting interrupts. Left as a todo since we
9803	* currently don't use activity states but ACTIVE. */
9804
9805	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9806	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9807
9808	/* Guest interruptibility-state. */
9809	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9810	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9811	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9812	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9813	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9814	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9815	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9816	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9817	{
9818	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9819	{
9820	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9821	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9822	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9823	}
9824	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9825	{
9826	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9827	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9828	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9829	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9830	}
9831	}
9832	/** @todo Assumes the processor is not in SMM. */
9833	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9834	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9835	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9836	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9837	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9838	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9839	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9840	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9841	{
9842	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9843	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9844	}
9845
9846	/* Pending debug exceptions. */
9847	#if HC_ARCH_BITS == 64
9848	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9849	AssertRCBreak(rc);
9850	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9851	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9852	u32Val = u64Val; /* For pending debug exceptions checks below. */
9853	#else
9854	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9855	AssertRCBreak(rc);
9856	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9857	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9858	#endif
9859
9860	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9861	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9862	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9863	{
9864	if ( (u32Eflags & X86_EFL_TF)
9865	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9866	{
9867	/* Bit 14 is PendingDebug.BS. */
9868	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9869	}
9870	if ( !(u32Eflags & X86_EFL_TF)
9871	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9872	{
9873	/* Bit 14 is PendingDebug.BS. */
9874	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9875	}
9876	}
9877
9878	/* VMCS link pointer. */
9879	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9880	AssertRCBreak(rc);
9881	if (u64Val != UINT64_C(0xffffffffffffffff))
9882	{
9883	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9884	/** @todo Bits beyond the processor's physical-address width MBZ. */
9885	/** @todo 32-bit located in memory referenced by value of this field (as a
9886	* physical address) must contain the processor's VMCS revision ID. */
9887	/** @todo SMM checks. */
9888	}
9889
9890	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9891	* not using nested paging? */
9892	if ( pVM->hm.s.fNestedPaging
9893	&& !fLongModeGuest
9894	&& CPUMIsGuestInPAEModeEx(pCtx))
9895	{
9896	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9897	AssertRCBreak(rc);
9898	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9899
9900	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9901	AssertRCBreak(rc);
9902	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9903
9904	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9905	AssertRCBreak(rc);
9906	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9907
9908	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9909	AssertRCBreak(rc);
9910	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9911	}
9912
9913	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9914	if (uError == VMX_IGS_ERROR)
9915	uError = VMX_IGS_REASON_NOT_FOUND;
9916	} while (0);
9917
9918	pVCpu->hm.s.u32HMError = uError;
9919	return uError;
9920
9921	#undef HMVMX_ERROR_BREAK
9922	#undef HMVMX_CHECK_BREAK
9923	}
9924
9925
9926	/**
9927	* Setup the APIC-access page for virtualizing APIC access.
9928	*
9929	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9930	* this not done as part of exporting guest state, see @bugref{8721}.
9931	*
9932	* @returns VBox status code.
9933	* @param pVCpu The cross context virtual CPU structure.
9934	*/
9935	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9936	{
9937	PVM pVM = pVCpu->CTX_SUFF(pVM);
9938	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9939
9940	Assert(PDMHasApic(pVM));
9941	Assert(u64MsrApicBase);
9942
9943	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9944	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9945
9946	/* Unalias any existing mapping. */
9947	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9948	AssertRCReturn(rc, rc);
9949
9950	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9951	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9952	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9953	AssertRCReturn(rc, rc);
9954
9955	/* Update the per-VCPU cache of the APIC base MSR. */
9956	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9957	return VINF_SUCCESS;
9958	}
9959
9960
9961	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9962	/**
9963	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9964	* nested-guest using hardware-assisted VMX.
9965	*
9966	* @param pVCpu The cross context virtual CPU structure.
9967	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9968	* @param pVmcsInfoGst The guest VMCS info. object.
9969	*/
9970	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9971	{
9972	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9973	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9974	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9975	Assert(pu64MsrBitmapNstGst);
9976	Assert(pu64MsrBitmapGst);
9977	Assert(pu64MsrBitmap);
9978
9979	/*
9980	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9981	* MSR that is intercepted by the guest is also intercepted while executing the
9982	* nested-guest using hardware-assisted VMX.
9983	*/
9984	uint32_t const cbFrag = sizeof(uint64_t);
9985	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9986	for (uint32_t i = 0; i <= cFrags; i++)
9987	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9988	}
9989
9990
9991	/**
9992	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9993	* hardware-assisted VMX execution of the nested-guest.
9994	*
9995	* For a guest, we don't modify these controls once we set up the VMCS and hence
9996	* this function is never called.
9997	*
9998	* For nested-guests since the guest hypervisor provides these controls on every
9999	* nested-guest VM-entry and could potentially change them everytime we need to
10000	* merge them before every nested-guest VM-entry.
10001	*
10002	* @returns VBox status code.
10003	* @param pVCpu The cross context virtual CPU structure.
10004	*/
10005	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10006	{
10007	PVM pVM = pVCpu->CTX_SUFF(pVM);
10008	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10009	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10010	Assert(pVmcsNstGst);
10011
10012	/*
10013	* Merge the controls with the requirements of the guest VMCS.
10014	*
10015	* We do not need to validate the nested-guest VMX features specified in the
10016	* nested-guest VMCS with the features supported by the physical CPU as it's
10017	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10018	*
10019	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10020	* guest are derived from the VMX features supported by the physical CPU.
10021	*/
10022
10023	/* Pin-based VM-execution controls. */
10024	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10025
10026	/* Processor-based VM-execution controls. */
10027	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10028	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10029	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10030	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10031	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10032
10033	/* Secondary processor-based VM-execution controls. */
10034	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10035	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10036	\| VMX_PROC_CTLS2_INVPCID
10037	\| VMX_PROC_CTLS2_RDTSCP
10038	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10039	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10040	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10041	\| VMX_PROC_CTLS2_VMFUNC));
10042
10043	/*
10044	* VM-entry controls:
10045	* These controls contains state that depends on the nested-guest state (primarily
10046	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
10047	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10048	* properly continue executing the nested-guest if the EFER MSR changes but does not
10049	* cause a nested-guest VM-exits.
10050	*
10051	* VM-exit controls:
10052	* These controls specify the host state on return. We cannot use the controls from
10053	* the nested-hypervisor state as is as it would contain the guest state rather than
10054	* the host state. Since the host state is subject to change (e.g. preemption, trips
10055	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10056	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10057	*
10058	* VM-entry MSR-load:
10059	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10060	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10061	*
10062	* VM-exit MSR-store:
10063	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10064	* context back into the VM-exit MSR-store area.
10065	*
10066	* VM-exit MSR-load areas:
10067	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10068	* we can entirely ignore what the nested-hypervisor wants to load here.
10069	*/
10070
10071	/*
10072	* Exception bitmap.
10073	*
10074	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10075	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10076	* keep the code more flexible if intercepting exceptions become more dynamic in
10077	* the future we do it as part of exporting the nested-guest state.
10078	*/
10079	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10080
10081	/*
10082	* CR0/CR4 guest/host mask.
10083	*
10084	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10085	* must cause VM-exits, so we need to merge them here.
10086	*/
10087	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10088	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10089
10090	/*
10091	* Page-fault error-code mask and match.
10092	*
10093	* Although we require unrestricted guest execution (and thereby nested-paging) for
10094	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10095	* normally intercept #PFs, it might intercept them for debugging purposes.
10096	*
10097	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10098	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10099	*/
10100	uint32_t u32XcptPFMask;
10101	uint32_t u32XcptPFMatch;
10102	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10103	{
10104	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10105	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10106	}
10107	else
10108	{
10109	u32XcptPFMask = 0;
10110	u32XcptPFMatch = 0;
10111	}
10112
10113	/*
10114	* Pause-Loop exiting.
10115	*/
10116	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10117	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10118
10119	/*
10120	* I/O Bitmap.
10121	*
10122	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10123	* always intercept all I/O port accesses.
10124	*/
10125	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10126
10127	/*
10128	* APIC-access page.
10129	*
10130	* The APIC-access page address has already been initialized while setting up the
10131	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10132	* should not be on any consequence to the host or to the guest for that matter, but
10133	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10134	* emulation to keep it simple.
10135	*/
10136
10137	/*
10138	* Virtual-APIC page and TPR threshold.
10139	*
10140	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10141	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10142	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10143	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10144	*/
10145	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10146	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10147	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10148	{
10149	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10150
10151	/*
10152	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10153	* we would fail to obtain a valid host-physical address for its guest-physical
10154	* address.
10155	*
10156	* We currently do not support this scenario. Maybe in the future if there is a
10157	* pressing need we can explore making this particular set of conditions work.
10158	* Right now we just cause a VM-entry failure.
10159	*
10160	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10161	* so should not really failure at the moment.
10162	*/
10163	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10164	}
10165	else
10166	{
10167	/*
10168	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10169	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10170	* be taken care of by EPT/shadow paging.
10171	*/
10172	if (pVM->hm.s.fAllow64BitGuests)
10173	{
10174	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10175	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10176	}
10177	}
10178
10179	/*
10180	* Validate basic assumptions.
10181	*/
10182	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10183	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10184	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10185	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10186
10187	/*
10188	* Commit it to the nested-guest VMCS.
10189	*/
10190	int rc = VINF_SUCCESS;
10191	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10192	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10193	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10194	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10195	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10196	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10197	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10198	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10199	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10200	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10201	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10202	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10203	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10204	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10205	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10206	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10207	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10208	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10209	{
10210	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10211	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10212	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10213	}
10214	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10215	{
10216	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10217	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10218	}
10219	AssertRCReturn(rc, rc);
10220
10221	/*
10222	* Update the nested-guest VMCS cache.
10223	*/
10224	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10225	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10226	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10227	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10228	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10229	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10230	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10231	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10232	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10233
10234	/*
10235	* MSR bitmap.
10236	*
10237	* The MSR bitmap address has already been initialized while setting up the
10238	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10239	*/
10240	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10241	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10242
10243	return VINF_SUCCESS;
10244	}
10245	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10246
10247
10248	/**
10249	* Does the preparations before executing guest code in VT-x.
10250	*
10251	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10252	* recompiler/IEM. We must be cautious what we do here regarding committing
10253	* guest-state information into the VMCS assuming we assuredly execute the
10254	* guest in VT-x mode.
10255	*
10256	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10257	* the common-state (TRPM/forceflags), we must undo those changes so that the
10258	* recompiler/IEM can (and should) use them when it resumes guest execution.
10259	* Otherwise such operations must be done when we can no longer exit to ring-3.
10260	*
10261	* @returns Strict VBox status code (i.e. informational status codes too).
10262	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10263	* have been disabled.
10264	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10265	* double-fault into the guest.
10266	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10267	* dispatched directly.
10268	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10269	*
10270	* @param pVCpu The cross context virtual CPU structure.
10271	* @param pVmxTransient The VMX-transient structure.
10272	* @param fStepping Whether we are single-stepping the guest in the
10273	* hypervisor debugger. Makes us ignore some of the reasons
10274	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10275	* if event dispatching took place.
10276	*/
10277	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10278	{
10279	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10280
10281	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10282	if (pVmxTransient->fIsNestedGuest)
10283	{
10284	RT_NOREF2(pVCpu, fStepping);
10285	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10286	return VINF_EM_RESCHEDULE_REM;
10287	}
10288	#endif
10289
10290	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10291	PGMRZDynMapFlushAutoSet(pVCpu);
10292	#endif
10293
10294	/*
10295	* Check and process force flag actions, some of which might require us to go back to ring-3.
10296	*/
10297	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10298	if (rcStrict == VINF_SUCCESS)
10299	{ /* FFs don't get set all the time. */ }
10300	else
10301	return rcStrict;
10302
10303	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10304	/*
10305	* Switch to the nested-guest VMCS as we may have transitioned into executing
10306	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10307	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10308	*
10309	* We do this as late as possible to minimize (though not completely remove)
10310	* clearing/loading VMCS again due to premature trips to ring-3 above.
10311	*/
10312	if (pVmxTransient->fIsNestedGuest)
10313	{
10314	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10315	{
10316	/*
10317	* Ensure we have synced everything from the guest VMCS and also flag that
10318	* that we need to export the full (nested) guest-CPU context to the
10319	* nested-guest VMCS.
10320	*/
10321	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10322	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10323
10324	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10325	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10326	if (RT_LIKELY(rc == VINF_SUCCESS))
10327	{
10328	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10329	ASMSetFlags(fEFlags);
10330	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10331
10332	/*
10333	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10334	* flag that we need to update the host MSR values there. Even if we decide
10335	* in the future to share the VM-exit MSR-store area page with the guest,
10336	* if its content differs, we would have to update the host MSRs anyway.
10337	*/
10338	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10339	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10340	}
10341	else
10342	{
10343	ASMSetFlags(fEFlags);
10344	return rc;
10345	}
10346	}
10347
10348	/*
10349	* Merge guest VMCS controls with the nested-guest VMCS controls.
10350	*
10351	* Even if we have not executed the guest prior to this (e.g. when resuming
10352	* from a saved state), we should be okay with merging controls as we
10353	* initialize the guest VMCS controls as part of VM setup phase.
10354	*/
10355	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10356	{
10357	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10358	AssertRCReturn(rc, rc);
10359	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10360	}
10361	}
10362	#endif
10363
10364	/*
10365	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10366	* We look at the guest VMCS control here as we always set it when supported by
10367	* the physical CPU. Looking at the nested-guest control here would not be
10368	* possible because they are not merged yet.
10369	*/
10370	PVM pVM = pVCpu->CTX_SUFF(pVM);
10371	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10372	Assert(pVmcsInfo);
10373	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10374	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10375	&& PDMHasApic(pVM))
10376	{
10377	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10378	AssertRCReturn(rc, rc);
10379	}
10380
10381	/*
10382	* Evaluate events to be injected into the guest.
10383	*
10384	* Events in TRPM can be injected without inspecting the guest state.
10385	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10386	* guest to cause a VM-exit the next time they are ready to receive the event.
10387	*/
10388	if (TRPMHasTrap(pVCpu))
10389	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10390
10391	uint32_t fIntrState;
10392	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10393
10394	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10395	/*
10396	* While evaluating pending events if something failed (unlikely) or if we were
10397	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10398	*/
10399	if ( rcStrict != VINF_SUCCESS
10400	\|\| ( pVmxTransient->fIsNestedGuest
10401	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10402	return rcStrict;
10403	#endif
10404
10405	/*
10406	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10407	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10408	* also result in triple-faulting the VM.
10409	*
10410	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10411	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10412	*/
10413	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10414	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10415	{ /* likely */ }
10416	else
10417	{
10418	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10419	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10420	return rcStrict;
10421	}
10422
10423	/*
10424	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10425	* import CR3 themselves. We will need to update them here, as even as late as the above
10426	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10427	* the below force flags to be set.
10428	*/
10429	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10430	{
10431	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10432	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10433	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10434	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10435	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10436	}
10437	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10438	{
10439	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10440	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10441	}
10442
10443	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10444	/* Paranoia. */
10445	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10446	#endif
10447
10448	/*
10449	* No longjmps to ring-3 from this point on!!!
10450	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10451	* This also disables flushing of the R0-logger instance (if any).
10452	*/
10453	VMMRZCallRing3Disable(pVCpu);
10454
10455	/*
10456	* Export the guest state bits.
10457	*
10458	* We cannot perform longjmps while loading the guest state because we do not preserve the
10459	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10460	* CPU migration.
10461	*
10462	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10463	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10464	*/
10465	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10466	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10467	{ /* likely */ }
10468	else
10469	{
10470	VMMRZCallRing3Enable(pVCpu);
10471	return rcStrict;
10472	}
10473
10474	/*
10475	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10476	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10477	* preemption disabled for a while. Since this is purely to aid the
10478	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10479	* disable interrupt on NT.
10480	*
10481	* We need to check for force-flags that could've possible been altered since we last
10482	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10483	* see @bugref{6398}).
10484	*
10485	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10486	* to ring-3 before executing guest code.
10487	*/
10488	pVmxTransient->fEFlags = ASMIntDisableFlags();
10489
10490	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10491	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10492	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10493	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10494	{
10495	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10496	{
10497	pVCpu->hm.s.Event.fPending = false;
10498
10499	/*
10500	* We've injected any pending events. This is really the point of no return (to ring-3).
10501	*
10502	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10503	* returns from this function, so don't enable them here.
10504	*/
10505	return VINF_SUCCESS;
10506	}
10507
10508	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10509	rcStrict = VINF_EM_RAW_INTERRUPT;
10510	}
10511	else
10512	{
10513	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10514	rcStrict = VINF_EM_RAW_TO_R3;
10515	}
10516
10517	ASMSetFlags(pVmxTransient->fEFlags);
10518	VMMRZCallRing3Enable(pVCpu);
10519
10520	return rcStrict;
10521	}
10522
10523
10524	/**
10525	* Final preparations before executing guest code using hardware-assisted VMX.
10526	*
10527	* We can no longer get preempted to a different host CPU and there are no returns
10528	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10529	* failures), this function is not intended to fail sans unrecoverable hardware
10530	* errors.
10531	*
10532	* @param pVCpu The cross context virtual CPU structure.
10533	* @param pVmxTransient The VMX-transient structure.
10534	*
10535	* @remarks Called with preemption disabled.
10536	* @remarks No-long-jump zone!!!
10537	*/
10538	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10539	{
10540	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10541	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10542	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10543	Assert(!pVCpu->hm.s.Event.fPending);
10544
10545	/*
10546	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10547	*/
10548	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10549	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10550
10551	PVM pVM = pVCpu->CTX_SUFF(pVM);
10552	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10553
10554	if (!CPUMIsGuestFPUStateActive(pVCpu))
10555	{
10556	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10557	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10558	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10559	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10560	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10561	}
10562
10563	/*
10564	* Re-save the host state bits as we may've been preempted (only happens when
10565	* thread-context hooks are used or when the VM start function changes).
10566	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10567	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10568	* see @bugref{8432}.
10569	*
10570	* This may also happen when switching to/from a nested-guest VMCS without leaving
10571	* ring-0.
10572	*/
10573	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10574	{
10575	int rc = hmR0VmxExportHostState(pVCpu);
10576	AssertRC(rc);
10577	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10578	}
10579	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10580
10581	/*
10582	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10583	*/
10584	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10585	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10586	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10587
10588	/*
10589	* Store status of the shared guest/host debug state at the time of VM-entry.
10590	*/
10591	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10592	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10593	{
10594	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10595	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10596	}
10597	else
10598	#endif
10599	{
10600	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10601	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10602	}
10603
10604	/*
10605	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10606	* more than one conditional check. The post-run side of our code shall determine
10607	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10608	*/
10609	if (pVmcsInfo->pbVirtApic)
10610	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10611
10612	/*
10613	* Update the host MSRs values in the VM-exit MSR-load area.
10614	*/
10615	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10616	{
10617	if (pVmcsInfo->cExitMsrLoad > 0)
10618	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10619	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10620	}
10621
10622	/*
10623	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10624	* VMX-preemption timer based on the next virtual sync clock deadline.
10625	*/
10626	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10627	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10628	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10629	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10630	{
10631	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10632	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10633	}
10634
10635	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10636	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10637	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10638	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10639
10640	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10641
10642	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10643	as we're about to start executing the guest . */
10644
10645	/*
10646	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10647	*
10648	* This is done this late as updating the TSC offsetting/preemption timer above
10649	* figures out if we can skip intercepting RDTSCP by calculating the number of
10650	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10651	*/
10652	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10653	{
10654	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10655	{
10656	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10657	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10658	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10659	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10660	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10661	AssertRC(rc);
10662	}
10663	else
10664	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10665	}
10666
10667	#ifdef VBOX_STRICT
10668	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10669	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10670	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10671	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10672	#endif
10673
10674	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10675	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10676	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10677	* see @bugref{9180#c54}. */
10678	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10679	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10680	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10681	#endif
10682	}
10683
10684
10685	/**
10686	* First C routine invoked after running guest code using hardware-assisted VMX.
10687	*
10688	* @param pVCpu The cross context virtual CPU structure.
10689	* @param pVmxTransient The VMX-transient structure.
10690	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10691	*
10692	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10693	*
10694	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10695	* unconditionally when it is safe to do so.
10696	*/
10697	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10698	{
10699	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10700
10701	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10702	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10703	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10704	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10705	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10706	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10707
10708	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10709	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10710	{
10711	uint64_t uGstTsc;
10712	if (!pVmxTransient->fIsNestedGuest)
10713	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10714	else
10715	{
10716	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10717	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10718	}
10719	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10720	}
10721
10722	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10723	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10724	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10725
10726	#if HC_ARCH_BITS == 64
10727	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10728	#endif
10729	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10730	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10731	and we need to leave it alone here. */
10732	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10733	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10734	#else
10735	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10736	#endif
10737	#ifdef VBOX_STRICT
10738	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10739	#endif
10740	Assert(!ASMIntAreEnabled());
10741	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10742	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10743
10744	/*
10745	* Save the basic VM-exit reason and check if the VM-entry failed.
10746	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10747	*/
10748	uint32_t uExitReason;
10749	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10750	AssertRC(rc);
10751	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10752	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10753
10754	/*
10755	* Check if VMLAUNCH/VMRESUME succeeded.
10756	* If this failed, we cause a guru meditation and cease further execution.
10757	*
10758	* However, if we are executing a nested-guest we might fail if we use the
10759	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10760	*/
10761	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10762	{
10763	/*
10764	* Update the VM-exit history array here even if the VM-entry failed due to:
10765	* - Invalid guest state.
10766	* - MSR loading.
10767	* - Machine-check event.
10768	*
10769	* In any of the above cases we will still have a "valid" VM-exit reason
10770	* despite @a fVMEntryFailed being false.
10771	*
10772	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10773	*
10774	* Note! We don't have CS or RIP at this point. Will probably address that later
10775	* by amending the history entry added here.
10776	*/
10777	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10778	UINT64_MAX, uHostTsc);
10779
10780	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10781	{
10782	VMMRZCallRing3Enable(pVCpu);
10783
10784	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10785	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10786
10787	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10788	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10789	AssertRC(rc);
10790	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10791	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10792	AssertRC(rc);
10793	#else
10794	/*
10795	* Import the guest-interruptibility state always as we need it while evaluating
10796	* injecting events on re-entry.
10797	*
10798	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10799	* checking for real-mode while exporting the state because all bits that cause
10800	* mode changes wrt CR0 are intercepted.
10801	*/
10802	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10803	AssertRC(rc);
10804	#endif
10805
10806	/*
10807	* Sync the TPR shadow with our APIC state.
10808	*/
10809	if ( !pVmxTransient->fIsNestedGuest
10810	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10811	{
10812	Assert(pVmcsInfo->pbVirtApic);
10813	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10814	{
10815	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10816	AssertRC(rc);
10817	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10818	}
10819	}
10820
10821	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10822	return;
10823	}
10824	}
10825	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10826	else if (pVmxTransient->fIsNestedGuest)
10827	{
10828	# if 0
10829	/*
10830	* Copy the VM-instruction error field to the guest VMCS.
10831	*/
10832	/** @todo NSTVMX: Verify we're using the fast path. */
10833	uint32_t u32RoVmInstrError;
10834	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
10835	AssertRCReturn(rc, rc);
10836	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10837	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
10838	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
10839	# else
10840	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
10841	# endif
10842	}
10843	#endif
10844	else
10845	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10846
10847	VMMRZCallRing3Enable(pVCpu);
10848	}
10849
10850
10851	/**
10852	* Runs the guest code using hardware-assisted VMX the normal way.
10853	*
10854	* @returns VBox status code.
10855	* @param pVCpu The cross context virtual CPU structure.
10856	* @param pcLoops Pointer to the number of executed loops.
10857	*/
10858	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10859	{
10860	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10861	Assert(pcLoops);
10862	Assert(*pcLoops <= cMaxResumeLoops);
10863
10864	VMXTRANSIENT VmxTransient;
10865	RT_ZERO(VmxTransient);
10866	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10867
10868	/* Paranoia. */
10869	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10870	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10871
10872	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10873	for (;;)
10874	{
10875	Assert(!HMR0SuspendPending());
10876	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10877	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10878
10879	/*
10880	* Preparatory work for running nested-guest code, this may force us to
10881	* return to ring-3.
10882	*
10883	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10884	*/
10885	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10886	if (rcStrict != VINF_SUCCESS)
10887	break;
10888
10889	/* Interrupts are disabled at this point! */
10890	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10891	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10892	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10893	/* Interrupts are re-enabled at this point! */
10894
10895	/*
10896	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10897	*/
10898	if (RT_SUCCESS(rcRun))
10899	{ /* very likely */ }
10900	else
10901	{
10902	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10903	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10904	return rcRun;
10905	}
10906
10907	/*
10908	* Profile the VM-exit.
10909	*/
10910	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10912	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10913	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10914	HMVMX_START_EXIT_DISPATCH_PROF();
10915
10916	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10917
10918	/*
10919	* Handle the VM-exit.
10920	*/
10921	#ifdef HMVMX_USE_FUNCTION_TABLE
10922	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10923	#else
10924	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10925	#endif
10926	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10927	if (rcStrict == VINF_SUCCESS)
10928	{
10929	if (++(*pcLoops) <= cMaxResumeLoops)
10930	continue;
10931	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10932	rcStrict = VINF_EM_RAW_INTERRUPT;
10933	}
10934	break;
10935	}
10936
10937	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10938	return rcStrict;
10939	}
10940
10941	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10942	/**
10943	* Runs the nested-guest code using hardware-assisted VMX.
10944	*
10945	* @returns VBox status code.
10946	* @param pVCpu The cross context virtual CPU structure.
10947	* @param pcLoops Pointer to the number of executed loops.
10948	*
10949	* @sa hmR0VmxRunGuestCodeNormal.
10950	*/
10951	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10952	{
10953	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10954	Assert(pcLoops);
10955	Assert(*pcLoops <= cMaxResumeLoops);
10956
10957	VMXTRANSIENT VmxTransient;
10958	RT_ZERO(VmxTransient);
10959	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10960	VmxTransient.fIsNestedGuest = true;
10961
10962	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10963	for (;;)
10964	{
10965	Assert(!HMR0SuspendPending());
10966	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10967	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10968
10969	/*
10970	* Preparatory work for running guest code, this may force us to
10971	* return to ring-3.
10972	*
10973	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10974	*/
10975	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10976	if (rcStrict != VINF_SUCCESS)
10977	break;
10978
10979	/* Interrupts are disabled at this point! */
10980	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10981	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10982	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10983	/* Interrupts are re-enabled at this point! */
10984
10985	/*
10986	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10987	*/
10988	if (RT_SUCCESS(rcRun))
10989	{ /* very likely */ }
10990	else
10991	{
10992	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10993	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10994	return rcRun;
10995	}
10996
10997	/*
10998	* Profile the VM-exit.
10999	*/
11000	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11002	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11003	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11004	HMVMX_START_EXIT_DISPATCH_PROF();
11005
11006	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11007
11008	/*
11009	* Handle the VM-exit.
11010	*/
11011	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11012	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11013	if ( rcStrict == VINF_SUCCESS
11014	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11015	{
11016	if (++(*pcLoops) <= cMaxResumeLoops)
11017	continue;
11018	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11019	rcStrict = VINF_EM_RAW_INTERRUPT;
11020	}
11021	break;
11022	}
11023
11024	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11025	return rcStrict;
11026	}
11027	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11028
11029
11030	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11031	* probes.
11032	*
11033	* The following few functions and associated structure contains the bloat
11034	* necessary for providing detailed debug events and dtrace probes as well as
11035	* reliable host side single stepping. This works on the principle of
11036	* "subclassing" the normal execution loop and workers. We replace the loop
11037	* method completely and override selected helpers to add necessary adjustments
11038	* to their core operation.
11039	*
11040	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11041	* any performance for debug and analysis features.
11042	*
11043	* @{
11044	*/
11045
11046	/**
11047	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11048	* the debug run loop.
11049	*/
11050	typedef struct VMXRUNDBGSTATE
11051	{
11052	/** The RIP we started executing at. This is for detecting that we stepped. */
11053	uint64_t uRipStart;
11054	/** The CS we started executing with. */
11055	uint16_t uCsStart;
11056
11057	/** Whether we've actually modified the 1st execution control field. */
11058	bool fModifiedProcCtls : 1;
11059	/** Whether we've actually modified the 2nd execution control field. */
11060	bool fModifiedProcCtls2 : 1;
11061	/** Whether we've actually modified the exception bitmap. */
11062	bool fModifiedXcptBitmap : 1;
11063
11064	/** We desire the modified the CR0 mask to be cleared. */
11065	bool fClearCr0Mask : 1;
11066	/** We desire the modified the CR4 mask to be cleared. */
11067	bool fClearCr4Mask : 1;
11068	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11069	uint32_t fCpe1Extra;
11070	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11071	uint32_t fCpe1Unwanted;
11072	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11073	uint32_t fCpe2Extra;
11074	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11075	uint32_t bmXcptExtra;
11076	/** The sequence number of the Dtrace provider settings the state was
11077	* configured against. */
11078	uint32_t uDtraceSettingsSeqNo;
11079	/** VM-exits to check (one bit per VM-exit). */
11080	uint32_t bmExitsToCheck[3];
11081
11082	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11083	uint32_t fProcCtlsInitial;
11084	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11085	uint32_t fProcCtls2Initial;
11086	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11087	uint32_t bmXcptInitial;
11088	} VMXRUNDBGSTATE;
11089	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11090	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11091
11092
11093	/**
11094	* Initializes the VMXRUNDBGSTATE structure.
11095	*
11096	* @param pVCpu The cross context virtual CPU structure of the
11097	* calling EMT.
11098	* @param pVmxTransient The VMX-transient structure.
11099	* @param pDbgState The debug state to initialize.
11100	*/
11101	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11102	{
11103	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11104	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11105
11106	pDbgState->fModifiedProcCtls = false;
11107	pDbgState->fModifiedProcCtls2 = false;
11108	pDbgState->fModifiedXcptBitmap = false;
11109	pDbgState->fClearCr0Mask = false;
11110	pDbgState->fClearCr4Mask = false;
11111	pDbgState->fCpe1Extra = 0;
11112	pDbgState->fCpe1Unwanted = 0;
11113	pDbgState->fCpe2Extra = 0;
11114	pDbgState->bmXcptExtra = 0;
11115	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11116	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11117	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11118	}
11119
11120
11121	/**
11122	* Updates the VMSC fields with changes requested by @a pDbgState.
11123	*
11124	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11125	* immediately before executing guest code, i.e. when interrupts are disabled.
11126	* We don't check status codes here as we cannot easily assert or return in the
11127	* latter case.
11128	*
11129	* @param pVCpu The cross context virtual CPU structure.
11130	* @param pVmxTransient The VMX-transient structure.
11131	* @param pDbgState The debug state.
11132	*/
11133	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11134	{
11135	/*
11136	* Ensure desired flags in VMCS control fields are set.
11137	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11138	*
11139	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11140	* there should be no stale data in pCtx at this point.
11141	*/
11142	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11143	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11144	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11145	{
11146	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11147	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11148	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11149	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11150	pDbgState->fModifiedProcCtls = true;
11151	}
11152
11153	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11154	{
11155	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11156	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11157	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11158	pDbgState->fModifiedProcCtls2 = true;
11159	}
11160
11161	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11162	{
11163	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11164	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11165	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11166	pDbgState->fModifiedXcptBitmap = true;
11167	}
11168
11169	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11170	{
11171	pVmcsInfo->u64Cr0Mask = 0;
11172	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11173	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11174	}
11175
11176	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11177	{
11178	pVmcsInfo->u64Cr4Mask = 0;
11179	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11180	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11181	}
11182
11183	NOREF(pVCpu);
11184	}
11185
11186
11187	/**
11188	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11189	* re-entry next time around.
11190	*
11191	* @returns Strict VBox status code (i.e. informational status codes too).
11192	* @param pVCpu The cross context virtual CPU structure.
11193	* @param pVmxTransient The VMX-transient structure.
11194	* @param pDbgState The debug state.
11195	* @param rcStrict The return code from executing the guest using single
11196	* stepping.
11197	*/
11198	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11199	VBOXSTRICTRC rcStrict)
11200	{
11201	/*
11202	* Restore VM-exit control settings as we may not reenter this function the
11203	* next time around.
11204	*/
11205	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11206
11207	/* We reload the initial value, trigger what we can of recalculations the
11208	next time around. From the looks of things, that's all that's required atm. */
11209	if (pDbgState->fModifiedProcCtls)
11210	{
11211	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11212	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11213	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11214	AssertRCReturn(rc2, rc2);
11215	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11216	}
11217
11218	/* We're currently the only ones messing with this one, so just restore the
11219	cached value and reload the field. */
11220	if ( pDbgState->fModifiedProcCtls2
11221	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11222	{
11223	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11224	AssertRCReturn(rc2, rc2);
11225	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11226	}
11227
11228	/* If we've modified the exception bitmap, we restore it and trigger
11229	reloading and partial recalculation the next time around. */
11230	if (pDbgState->fModifiedXcptBitmap)
11231	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11232
11233	return rcStrict;
11234	}
11235
11236
11237	/**
11238	* Configures VM-exit controls for current DBGF and DTrace settings.
11239	*
11240	* This updates @a pDbgState and the VMCS execution control fields to reflect
11241	* the necessary VM-exits demanded by DBGF and DTrace.
11242	*
11243	* @param pVCpu The cross context virtual CPU structure.
11244	* @param pVmxTransient The VMX-transient structure. May update
11245	* fUpdatedTscOffsettingAndPreemptTimer.
11246	* @param pDbgState The debug state.
11247	*/
11248	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11249	{
11250	/*
11251	* Take down the dtrace serial number so we can spot changes.
11252	*/
11253	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11254	ASMCompilerBarrier();
11255
11256	/*
11257	* We'll rebuild most of the middle block of data members (holding the
11258	* current settings) as we go along here, so start by clearing it all.
11259	*/
11260	pDbgState->bmXcptExtra = 0;
11261	pDbgState->fCpe1Extra = 0;
11262	pDbgState->fCpe1Unwanted = 0;
11263	pDbgState->fCpe2Extra = 0;
11264	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11265	pDbgState->bmExitsToCheck[i] = 0;
11266
11267	/*
11268	* Software interrupts (INT XXh) - no idea how to trigger these...
11269	*/
11270	PVM pVM = pVCpu->CTX_SUFF(pVM);
11271	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11272	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11273	{
11274	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11275	}
11276
11277	/*
11278	* INT3 breakpoints - triggered by #BP exceptions.
11279	*/
11280	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11281	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11282
11283	/*
11284	* Exception bitmap and XCPT events+probes.
11285	*/
11286	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11287	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11288	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11289
11290	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11291	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11292	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11293	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11294	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11295	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11296	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11297	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11298	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11299	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11300	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11301	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11302	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11303	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11304	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11305	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11306	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11307	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11308
11309	if (pDbgState->bmXcptExtra)
11310	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11311
11312	/*
11313	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11314	*
11315	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11316	* So, when adding/changing/removing please don't forget to update it.
11317	*
11318	* Some of the macros are picking up local variables to save horizontal space,
11319	* (being able to see it in a table is the lesser evil here).
11320	*/
11321	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11322	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11323	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11324	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11325	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11326	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11327	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11328	} else do { } while (0)
11329	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11330	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11331	{ \
11332	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11333	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11334	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11335	} else do { } while (0)
11336	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11337	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11338	{ \
11339	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11340	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11341	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11342	} else do { } while (0)
11343	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11344	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11345	{ \
11346	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11347	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11348	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11349	} else do { } while (0)
11350
11351	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11352	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11353	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11354	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11355	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11356
11357	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11358	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11359	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11360	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11361	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11362	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11363	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11364	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11365	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11366	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11367	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11368	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11369	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11370	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11371	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11372	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11373	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11374	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11375	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11376	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11377	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11378	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11379	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11380	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11381	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11382	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11383	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11384	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11385	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11386	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11387	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11388	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11389	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11390	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11391	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11392	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11393
11394	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11395	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11396	{
11397	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11398	\| CPUMCTX_EXTRN_APIC_TPR);
11399	AssertRC(rc);
11400
11401	#if 0 /** @todo fix me */
11402	pDbgState->fClearCr0Mask = true;
11403	pDbgState->fClearCr4Mask = true;
11404	#endif
11405	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11406	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11407	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11408	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11409	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11410	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11411	require clearing here and in the loop if we start using it. */
11412	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11413	}
11414	else
11415	{
11416	if (pDbgState->fClearCr0Mask)
11417	{
11418	pDbgState->fClearCr0Mask = false;
11419	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11420	}
11421	if (pDbgState->fClearCr4Mask)
11422	{
11423	pDbgState->fClearCr4Mask = false;
11424	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11425	}
11426	}
11427	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11428	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11429
11430	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11431	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11432	{
11433	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11434	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11435	}
11436	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11437	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11438
11439	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11440	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11441	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11442	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11443	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11444	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11445	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11446	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11447	#if 0 /** @todo too slow, fix handler. */
11448	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11449	#endif
11450	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11451
11452	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11453	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11454	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11455	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11456	{
11457	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11458	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11459	}
11460	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11461	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11462	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11463	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11464
11465	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11466	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11467	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11468	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11469	{
11470	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11471	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11472	}
11473	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11474	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11475	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11476	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11477
11478	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11479	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11480	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11481	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11482	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11483	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11484	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11485	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11486	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11487	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11488	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11489	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11490	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11491	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11492	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11493	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11494	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11495	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11496	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11497	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11498	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11499	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11500
11501	#undef IS_EITHER_ENABLED
11502	#undef SET_ONLY_XBM_IF_EITHER_EN
11503	#undef SET_CPE1_XBM_IF_EITHER_EN
11504	#undef SET_CPEU_XBM_IF_EITHER_EN
11505	#undef SET_CPE2_XBM_IF_EITHER_EN
11506
11507	/*
11508	* Sanitize the control stuff.
11509	*/
11510	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11511	if (pDbgState->fCpe2Extra)
11512	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11513	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11514	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11515	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11516	{
11517	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11518	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11519	}
11520
11521	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11522	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11523	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11524	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11525	}
11526
11527
11528	/**
11529	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11530	* appropriate.
11531	*
11532	* The caller has checked the VM-exit against the
11533	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11534	* already, so we don't have to do that either.
11535	*
11536	* @returns Strict VBox status code (i.e. informational status codes too).
11537	* @param pVCpu The cross context virtual CPU structure.
11538	* @param pVmxTransient The VMX-transient structure.
11539	* @param uExitReason The VM-exit reason.
11540	*
11541	* @remarks The name of this function is displayed by dtrace, so keep it short
11542	* and to the point. No longer than 33 chars long, please.
11543	*/
11544	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11545	{
11546	/*
11547	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11548	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11549	*
11550	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11551	* does. Must add/change/remove both places. Same ordering, please.
11552	*
11553	* Added/removed events must also be reflected in the next section
11554	* where we dispatch dtrace events.
11555	*/
11556	bool fDtrace1 = false;
11557	bool fDtrace2 = false;
11558	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11559	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11560	uint32_t uEventArg = 0;
11561	#define SET_EXIT(a_EventSubName) \
11562	do { \
11563	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11564	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11565	} while (0)
11566	#define SET_BOTH(a_EventSubName) \
11567	do { \
11568	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11569	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11570	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11571	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11572	} while (0)
11573	switch (uExitReason)
11574	{
11575	case VMX_EXIT_MTF:
11576	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11577
11578	case VMX_EXIT_XCPT_OR_NMI:
11579	{
11580	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11581	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11582	{
11583	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11584	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11585	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11586	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11587	{
11588	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11589	{
11590	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11591	uEventArg = pVmxTransient->uExitIntErrorCode;
11592	}
11593	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11594	switch (enmEvent1)
11595	{
11596	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11597	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11598	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11599	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11600	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11601	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11602	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11603	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11604	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11605	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11606	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11607	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11608	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11609	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11610	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11611	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11612	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11613	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11614	default: break;
11615	}
11616	}
11617	else
11618	AssertFailed();
11619	break;
11620
11621	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11622	uEventArg = idxVector;
11623	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11624	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11625	break;
11626	}
11627	break;
11628	}
11629
11630	case VMX_EXIT_TRIPLE_FAULT:
11631	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11632	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11633	break;
11634	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11635	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11636	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11637	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11638	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11639
11640	/* Instruction specific VM-exits: */
11641	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11642	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11643	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11644	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11645	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11646	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11647	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11648	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11649	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11650	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11651	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11652	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11653	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11654	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11655	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11656	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11657	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11658	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11659	case VMX_EXIT_MOV_CRX:
11660	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11661	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11662	SET_BOTH(CRX_READ);
11663	else
11664	SET_BOTH(CRX_WRITE);
11665	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11666	break;
11667	case VMX_EXIT_MOV_DRX:
11668	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11669	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11670	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11671	SET_BOTH(DRX_READ);
11672	else
11673	SET_BOTH(DRX_WRITE);
11674	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11675	break;
11676	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11677	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11678	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11679	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11680	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11681	case VMX_EXIT_GDTR_IDTR_ACCESS:
11682	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11683	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11684	{
11685	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11686	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11687	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11688	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11689	}
11690	break;
11691
11692	case VMX_EXIT_LDTR_TR_ACCESS:
11693	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11694	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11695	{
11696	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11697	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11698	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11699	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11700	}
11701	break;
11702
11703	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11704	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11705	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11706	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11707	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11708	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11709	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11710	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11711	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11712	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11713	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11714
11715	/* Events that aren't relevant at this point. */
11716	case VMX_EXIT_EXT_INT:
11717	case VMX_EXIT_INT_WINDOW:
11718	case VMX_EXIT_NMI_WINDOW:
11719	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11720	case VMX_EXIT_PREEMPT_TIMER:
11721	case VMX_EXIT_IO_INSTR:
11722	break;
11723
11724	/* Errors and unexpected events. */
11725	case VMX_EXIT_INIT_SIGNAL:
11726	case VMX_EXIT_SIPI:
11727	case VMX_EXIT_IO_SMI:
11728	case VMX_EXIT_SMI:
11729	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11730	case VMX_EXIT_ERR_MSR_LOAD:
11731	case VMX_EXIT_ERR_MACHINE_CHECK:
11732	case VMX_EXIT_PML_FULL:
11733	case VMX_EXIT_VIRTUALIZED_EOI:
11734	break;
11735
11736	default:
11737	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11738	break;
11739	}
11740	#undef SET_BOTH
11741	#undef SET_EXIT
11742
11743	/*
11744	* Dtrace tracepoints go first. We do them here at once so we don't
11745	* have to copy the guest state saving and stuff a few dozen times.
11746	* Down side is that we've got to repeat the switch, though this time
11747	* we use enmEvent since the probes are a subset of what DBGF does.
11748	*/
11749	if (fDtrace1 \|\| fDtrace2)
11750	{
11751	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11752	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11753	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11754	switch (enmEvent1)
11755	{
11756	/** @todo consider which extra parameters would be helpful for each probe. */
11757	case DBGFEVENT_END: break;
11758	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11759	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11760	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11761	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11762	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11763	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11764	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11765	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11766	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11767	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11768	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11769	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11770	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11771	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11772	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11773	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11774	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11775	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11776	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11777	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11778	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11779	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11780	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11781	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11782	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11783	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11784	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11785	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11786	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11787	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11788	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11789	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11790	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11791	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11792	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11793	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11794	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11795	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11796	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11797	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11798	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11799	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11800	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11801	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11802	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11803	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11804	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11805	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11806	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11807	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11808	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11809	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11810	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11811	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11812	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11813	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11814	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11815	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11816	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11817	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11818	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11819	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11820	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11821	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11822	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11823	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11824	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11825	}
11826	switch (enmEvent2)
11827	{
11828	/** @todo consider which extra parameters would be helpful for each probe. */
11829	case DBGFEVENT_END: break;
11830	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11831	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11832	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11833	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11834	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11835	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11836	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11837	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11838	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11839	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11840	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11841	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11842	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11843	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11844	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11845	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11846	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11847	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11848	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11849	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11850	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11851	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11852	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11853	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11854	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11855	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11856	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11857	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11858	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11859	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11860	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11861	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11862	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11863	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11864	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11865	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11866	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11867	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11868	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11869	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11870	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11871	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11872	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11873	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11874	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11875	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11876	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11877	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11878	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11879	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11880	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11881	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11882	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11883	}
11884	}
11885
11886	/*
11887	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11888	* the DBGF call will do a full check).
11889	*
11890	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11891	* Note! If we have to events, we prioritize the first, i.e. the instruction
11892	* one, in order to avoid event nesting.
11893	*/
11894	PVM pVM = pVCpu->CTX_SUFF(pVM);
11895	if ( enmEvent1 != DBGFEVENT_END
11896	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11897	{
11898	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11899	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11900	if (rcStrict != VINF_SUCCESS)
11901	return rcStrict;
11902	}
11903	else if ( enmEvent2 != DBGFEVENT_END
11904	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11905	{
11906	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11907	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11908	if (rcStrict != VINF_SUCCESS)
11909	return rcStrict;
11910	}
11911
11912	return VINF_SUCCESS;
11913	}
11914
11915
11916	/**
11917	* Single-stepping VM-exit filtering.
11918	*
11919	* This is preprocessing the VM-exits and deciding whether we've gotten far
11920	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11921	* handling is performed.
11922	*
11923	* @returns Strict VBox status code (i.e. informational status codes too).
11924	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11925	* @param pVmxTransient The VMX-transient structure.
11926	* @param pDbgState The debug state.
11927	*/
11928	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11929	{
11930	/*
11931	* Expensive (saves context) generic dtrace VM-exit probe.
11932	*/
11933	uint32_t const uExitReason = pVmxTransient->uExitReason;
11934	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11935	{ /* more likely */ }
11936	else
11937	{
11938	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11939	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11940	AssertRC(rc);
11941	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11942	}
11943
11944	/*
11945	* Check for host NMI, just to get that out of the way.
11946	*/
11947	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11948	{ /* normally likely */ }
11949	else
11950	{
11951	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11952	AssertRCReturn(rc2, rc2);
11953	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11954	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11955	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11956	}
11957
11958	/*
11959	* Check for single stepping event if we're stepping.
11960	*/
11961	if (pVCpu->hm.s.fSingleInstruction)
11962	{
11963	switch (uExitReason)
11964	{
11965	case VMX_EXIT_MTF:
11966	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11967
11968	/* Various events: */
11969	case VMX_EXIT_XCPT_OR_NMI:
11970	case VMX_EXIT_EXT_INT:
11971	case VMX_EXIT_TRIPLE_FAULT:
11972	case VMX_EXIT_INT_WINDOW:
11973	case VMX_EXIT_NMI_WINDOW:
11974	case VMX_EXIT_TASK_SWITCH:
11975	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11976	case VMX_EXIT_APIC_ACCESS:
11977	case VMX_EXIT_EPT_VIOLATION:
11978	case VMX_EXIT_EPT_MISCONFIG:
11979	case VMX_EXIT_PREEMPT_TIMER:
11980
11981	/* Instruction specific VM-exits: */
11982	case VMX_EXIT_CPUID:
11983	case VMX_EXIT_GETSEC:
11984	case VMX_EXIT_HLT:
11985	case VMX_EXIT_INVD:
11986	case VMX_EXIT_INVLPG:
11987	case VMX_EXIT_RDPMC:
11988	case VMX_EXIT_RDTSC:
11989	case VMX_EXIT_RSM:
11990	case VMX_EXIT_VMCALL:
11991	case VMX_EXIT_VMCLEAR:
11992	case VMX_EXIT_VMLAUNCH:
11993	case VMX_EXIT_VMPTRLD:
11994	case VMX_EXIT_VMPTRST:
11995	case VMX_EXIT_VMREAD:
11996	case VMX_EXIT_VMRESUME:
11997	case VMX_EXIT_VMWRITE:
11998	case VMX_EXIT_VMXOFF:
11999	case VMX_EXIT_VMXON:
12000	case VMX_EXIT_MOV_CRX:
12001	case VMX_EXIT_MOV_DRX:
12002	case VMX_EXIT_IO_INSTR:
12003	case VMX_EXIT_RDMSR:
12004	case VMX_EXIT_WRMSR:
12005	case VMX_EXIT_MWAIT:
12006	case VMX_EXIT_MONITOR:
12007	case VMX_EXIT_PAUSE:
12008	case VMX_EXIT_GDTR_IDTR_ACCESS:
12009	case VMX_EXIT_LDTR_TR_ACCESS:
12010	case VMX_EXIT_INVEPT:
12011	case VMX_EXIT_RDTSCP:
12012	case VMX_EXIT_INVVPID:
12013	case VMX_EXIT_WBINVD:
12014	case VMX_EXIT_XSETBV:
12015	case VMX_EXIT_RDRAND:
12016	case VMX_EXIT_INVPCID:
12017	case VMX_EXIT_VMFUNC:
12018	case VMX_EXIT_RDSEED:
12019	case VMX_EXIT_XSAVES:
12020	case VMX_EXIT_XRSTORS:
12021	{
12022	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12023	AssertRCReturn(rc, rc);
12024	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12025	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12026	return VINF_EM_DBG_STEPPED;
12027	break;
12028	}
12029
12030	/* Errors and unexpected events: */
12031	case VMX_EXIT_INIT_SIGNAL:
12032	case VMX_EXIT_SIPI:
12033	case VMX_EXIT_IO_SMI:
12034	case VMX_EXIT_SMI:
12035	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12036	case VMX_EXIT_ERR_MSR_LOAD:
12037	case VMX_EXIT_ERR_MACHINE_CHECK:
12038	case VMX_EXIT_PML_FULL:
12039	case VMX_EXIT_VIRTUALIZED_EOI:
12040	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12041	break;
12042
12043	default:
12044	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12045	break;
12046	}
12047	}
12048
12049	/*
12050	* Check for debugger event breakpoints and dtrace probes.
12051	*/
12052	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12053	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12054	{
12055	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12056	if (rcStrict != VINF_SUCCESS)
12057	return rcStrict;
12058	}
12059
12060	/*
12061	* Normal processing.
12062	*/
12063	#ifdef HMVMX_USE_FUNCTION_TABLE
12064	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12065	#else
12066	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12067	#endif
12068	}
12069
12070
12071	/**
12072	* Single steps guest code using hardware-assisted VMX.
12073	*
12074	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12075	* but single-stepping through the hypervisor debugger.
12076	*
12077	* @returns Strict VBox status code (i.e. informational status codes too).
12078	* @param pVCpu The cross context virtual CPU structure.
12079	* @param pcLoops Pointer to the number of executed loops.
12080	*
12081	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12082	*/
12083	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12084	{
12085	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12086	Assert(pcLoops);
12087	Assert(*pcLoops <= cMaxResumeLoops);
12088
12089	VMXTRANSIENT VmxTransient;
12090	RT_ZERO(VmxTransient);
12091	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12092
12093	/* Set HMCPU indicators. */
12094	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12095	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12096	pVCpu->hm.s.fDebugWantRdTscExit = false;
12097	pVCpu->hm.s.fUsingDebugLoop = true;
12098
12099	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12100	VMXRUNDBGSTATE DbgState;
12101	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12102	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12103
12104	/*
12105	* The loop.
12106	*/
12107	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12108	for (;;)
12109	{
12110	Assert(!HMR0SuspendPending());
12111	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12112	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12113	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12114
12115	/* Set up VM-execution controls the next two can respond to. */
12116	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12117
12118	/*
12119	* Preparatory work for running guest code, this may force us to
12120	* return to ring-3.
12121	*
12122	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12123	*/
12124	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12125	if (rcStrict != VINF_SUCCESS)
12126	break;
12127
12128	/* Interrupts are disabled at this point! */
12129	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12130
12131	/* Override any obnoxious code in the above two calls. */
12132	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12133
12134	/*
12135	* Finally execute the guest.
12136	*/
12137	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12138
12139	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12140	/* Interrupts are re-enabled at this point! */
12141
12142	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12143	if (RT_SUCCESS(rcRun))
12144	{ /* very likely */ }
12145	else
12146	{
12147	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12148	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12149	return rcRun;
12150	}
12151
12152	/* Profile the VM-exit. */
12153	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12155	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12156	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12157	HMVMX_START_EXIT_DISPATCH_PROF();
12158
12159	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12160
12161	/*
12162	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12163	*/
12164	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12165	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12166	if (rcStrict != VINF_SUCCESS)
12167	break;
12168	if (++(*pcLoops) > cMaxResumeLoops)
12169	{
12170	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12171	rcStrict = VINF_EM_RAW_INTERRUPT;
12172	break;
12173	}
12174
12175	/*
12176	* Stepping: Did the RIP change, if so, consider it a single step.
12177	* Otherwise, make sure one of the TFs gets set.
12178	*/
12179	if (fStepping)
12180	{
12181	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12182	AssertRC(rc);
12183	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12184	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12185	{
12186	rcStrict = VINF_EM_DBG_STEPPED;
12187	break;
12188	}
12189	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12190	}
12191
12192	/*
12193	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12194	*/
12195	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12196	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12197	}
12198
12199	/*
12200	* Clear the X86_EFL_TF if necessary.
12201	*/
12202	if (pVCpu->hm.s.fClearTrapFlag)
12203	{
12204	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12205	AssertRC(rc);
12206	pVCpu->hm.s.fClearTrapFlag = false;
12207	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12208	}
12209	/** @todo there seems to be issues with the resume flag when the monitor trap
12210	* flag is pending without being used. Seen early in bios init when
12211	* accessing APIC page in protected mode. */
12212
12213	/*
12214	* Restore VM-exit control settings as we may not re-enter this function the
12215	* next time around.
12216	*/
12217	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12218
12219	/* Restore HMCPU indicators. */
12220	pVCpu->hm.s.fUsingDebugLoop = false;
12221	pVCpu->hm.s.fDebugWantRdTscExit = false;
12222	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12223
12224	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12225	return rcStrict;
12226	}
12227
12228
12229	/** @} */
12230
12231
12232	/**
12233	* Checks if any expensive dtrace probes are enabled and we should go to the
12234	* debug loop.
12235	*
12236	* @returns true if we should use debug loop, false if not.
12237	*/
12238	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12239	{
12240	/* It's probably faster to OR the raw 32-bit counter variables together.
12241	Since the variables are in an array and the probes are next to one
12242	another (more or less), we have good locality. So, better read
12243	eight-nine cache lines ever time and only have one conditional, than
12244	128+ conditionals, right? */
12245	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12246	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12247	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12248	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12249	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12250	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12251	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12252	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12253	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12254	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12255	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12256	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12257	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12258	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12259	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12260	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12261	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12262	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12263	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12264	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12265	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12266	) != 0
12267	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12268	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12269	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12270	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12271	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12272	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12273	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12274	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12275	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12276	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12277	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12278	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12279	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12280	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12281	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12282	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12283	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12284	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12285	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12286	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12287	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12288	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12289	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12290	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12291	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12292	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12293	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12294	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12295	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12296	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12297	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12298	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12299	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12300	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12301	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12302	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12303	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12304	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12305	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12306	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12307	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12308	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12309	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12310	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12311	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12312	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12313	) != 0
12314	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12315	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12316	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12317	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12318	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12319	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12320	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12321	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12322	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12323	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12324	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12325	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12326	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12327	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12328	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12329	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12330	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12331	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12332	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12333	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12334	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12335	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12336	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12337	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12338	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12339	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12340	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12341	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12342	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12343	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12344	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12345	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12346	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12347	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12348	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12349	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12350	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12351	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12352	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12353	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12354	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12355	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12356	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12357	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12358	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12359	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12360	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12361	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12362	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12363	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12364	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12365	) != 0;
12366	}
12367
12368
12369	/**
12370	* Runs the guest using hardware-assisted VMX.
12371	*
12372	* @returns Strict VBox status code (i.e. informational status codes too).
12373	* @param pVCpu The cross context virtual CPU structure.
12374	*/
12375	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12376	{
12377	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12378	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12379	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12380	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12381
12382	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12383
12384	VBOXSTRICTRC rcStrict;
12385	uint32_t cLoops = 0;
12386	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12387	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12388	#else
12389	bool const fInNestedGuestMode = false;
12390	#endif
12391	if (!fInNestedGuestMode)
12392	{
12393	if ( !pVCpu->hm.s.fUseDebugLoop
12394	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12395	&& !DBGFIsStepping(pVCpu)
12396	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12397	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12398	else
12399	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12400	}
12401	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12402	else
12403	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12404
12405	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12406	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12407	#endif
12408
12409	if (rcStrict == VERR_EM_INTERPRETER)
12410	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12411	else if (rcStrict == VINF_EM_RESET)
12412	rcStrict = VINF_EM_TRIPLE_FAULT;
12413
12414	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12415	if (RT_FAILURE(rc2))
12416	{
12417	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12418	rcStrict = rc2;
12419	}
12420	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12421	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12422	return rcStrict;
12423	}
12424
12425
12426	#ifndef HMVMX_USE_FUNCTION_TABLE
12427	/**
12428	* Handles a guest VM-exit from hardware-assisted VMX execution.
12429	*
12430	* @returns Strict VBox status code (i.e. informational status codes too).
12431	* @param pVCpu The cross context virtual CPU structure.
12432	* @param pVmxTransient The VMX-transient structure.
12433	*/
12434	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12435	{
12436	#ifdef DEBUG_ramshankar
12437	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12438	do { \
12439	if (a_fSave != 0) \
12440	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12441	VBOXSTRICTRC rcStrict = a_CallExpr; \
12442	if (a_fSave != 0) \
12443	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12444	return rcStrict; \
12445	} while (0)
12446	#else
12447	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12448	#endif
12449	uint32_t const rcReason = pVmxTransient->uExitReason;
12450	switch (rcReason)
12451	{
12452	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12453	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12454	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12455	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12456	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12457	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12458	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12459	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12460	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12461	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12462	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12463	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12464	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12465	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12466	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12467	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12468	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12469	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12470	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12471	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12472	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12473	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12474	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12475	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12476	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12477	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12478	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12479	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12480	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12481	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12482	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12483	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12484	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12485	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12486	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12487	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12488	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12489	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12490	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12491	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12492	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12493	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12494	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12495	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12496	#else
12497	case VMX_EXIT_VMCLEAR:
12498	case VMX_EXIT_VMLAUNCH:
12499	case VMX_EXIT_VMPTRLD:
12500	case VMX_EXIT_VMPTRST:
12501	case VMX_EXIT_VMREAD:
12502	case VMX_EXIT_VMRESUME:
12503	case VMX_EXIT_VMWRITE:
12504	case VMX_EXIT_VMXOFF:
12505	case VMX_EXIT_VMXON:
12506	case VMX_EXIT_INVVPID:
12507	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12508	#endif
12509
12510	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12511	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12512	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12513
12514	case VMX_EXIT_RSM:
12515	case VMX_EXIT_RDSEED:
12516	case VMX_EXIT_ENCLS:
12517	case VMX_EXIT_INVEPT:
12518	case VMX_EXIT_VMFUNC:
12519	case VMX_EXIT_XSAVES:
12520	case VMX_EXIT_XRSTORS:
12521	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12522
12523	case VMX_EXIT_INIT_SIGNAL:
12524	case VMX_EXIT_SIPI:
12525	case VMX_EXIT_IO_SMI:
12526	case VMX_EXIT_SMI:
12527	case VMX_EXIT_ERR_MSR_LOAD:
12528	case VMX_EXIT_ERR_MACHINE_CHECK:
12529	case VMX_EXIT_PML_FULL:
12530	case VMX_EXIT_VIRTUALIZED_EOI:
12531	case VMX_EXIT_APIC_WRITE:
12532	default:
12533	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12534	}
12535	#undef VMEXIT_CALL_RET
12536	}
12537	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12538
12539
12540	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12541	/**
12542	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12543	*
12544	* @returns Strict VBox status code (i.e. informational status codes too).
12545	* @param pVCpu The cross context virtual CPU structure.
12546	* @param pVmxTransient The VMX-transient structure.
12547	*/
12548	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12549	{
12550	VBOXSTRICTRC rcStrict;
12551	uint32_t const uExitReason = pVmxTransient->uExitReason;
12552	switch (uExitReason)
12553	{
12554	case VMX_EXIT_EPT_MISCONFIG:
12555	rcStrict = hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12556	break;
12557
12558	case VMX_EXIT_EPT_VIOLATION:
12559	rcStrict = hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12560	break;
12561
12562	case VMX_EXIT_IO_INSTR:
12563	{
12564	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12565	AssertRCReturn(rc, rc);
12566
12567	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12568	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
12569	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
12570
12571	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
12572	uint8_t const cbAccess = s_aIOSizes[uIOSize];
12573	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
12574	{
12575	/*
12576	* IN/OUT instruction:
12577	* - Provides VM-exit instruction length.
12578	*
12579	* INS/OUTS instruction:
12580	* - Provides VM-exit instruction length.
12581	* - Provides Guest-linear address.
12582	* - Optionally provides VM-exit instruction info (depends on CPU feature).
12583	*/
12584	PVM pVM = pVCpu->CTX_SUFF(pVM);
12585	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12586	AssertRCReturn(rc, rc);
12587
12588	/* Make sure we don't use stale VMX-transient info. */
12589	pVmxTransient->ExitInstrInfo.u = 0;
12590	pVmxTransient->uGuestLinearAddr = 0;
12591
12592	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
12593	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12594	if (fIOString)
12595	{
12596	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12597	if (fVmxInsOutsInfo)
12598	{
12599	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
12600	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12601	}
12602	}
12603	AssertRCReturn(rc, rc);
12604
12605	VMXVEXITINFO ExitInfo;
12606	RT_ZERO(ExitInfo);
12607	ExitInfo.uReason = uExitReason;
12608	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12609	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12610	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12611	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
12612	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12613	}
12614	else
12615	rcStrict = hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
12616	break;
12617	}
12618
12619	case VMX_EXIT_HLT:
12620	{
12621	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12622	AssertRCReturn(rc, rc);
12623	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
12624	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12625	else
12626	rcStrict = hmR0VmxExitHlt(pVCpu, pVmxTransient);
12627	break;
12628	}
12629
12630	case VMX_EXIT_RDTSC:
12631	{
12632	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12633	AssertRCReturn(rc, rc);
12634	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12635	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12636	else
12637	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12638	break;
12639	}
12640
12641	case VMX_EXIT_RDTSCP:
12642	{
12643	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12644	AssertRCReturn(rc, rc);
12645	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
12646	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12647	else
12648	rcStrict = hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
12649	break;
12650	}
12651
12652	/*
12653	* Instructions that cause VM-exits unconditionally.
12654	* - Provides VM-exit instruction length ONLY.
12655	*/
12656	case VMX_EXIT_CPUID:
12657	case VMX_EXIT_VMCALL:
12658	case VMX_EXIT_GETSEC:
12659	case VMX_EXIT_INVD:
12660	case VMX_EXIT_XSETBV:
12661	case VMX_EXIT_VMLAUNCH:
12662	case VMX_EXIT_VMRESUME:
12663	case VMX_EXIT_VMXOFF:
12664	{
12665	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12666	AssertRCReturn(rc, rc);
12667	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12668	break;
12669	}
12670
12671	/*
12672	* Instructions that cause VM-exits unconditionally.
12673	* - Provides VM-exit instruction length.
12674	* - Provides VM-exit information.
12675	* - Optionally provides VM-exit qualification.
12676	*
12677	* Since VM-exit qualification is 0 for all VM-exits where it is not
12678	* applicable, reading and passing it to the guest should produce
12679	* defined behavior.
12680	*
12681	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12682	*/
12683	case VMX_EXIT_INVEPT:
12684	case VMX_EXIT_INVVPID:
12685	case VMX_EXIT_VMCLEAR:
12686	case VMX_EXIT_VMPTRLD:
12687	case VMX_EXIT_VMPTRST:
12688	case VMX_EXIT_VMXON:
12689	{
12690	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12691	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12692	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12693	AssertRCReturn(rc, rc);
12694
12695	VMXVEXITINFO ExitInfo;
12696	RT_ZERO(ExitInfo);
12697	ExitInfo.uReason = uExitReason;
12698	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12699	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12700	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12701	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12702	break;
12703	}
12704
12705	case VMX_EXIT_INVLPG:
12706	{
12707	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12708	{
12709	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12710	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12711	AssertRCReturn(rc, rc);
12712
12713	VMXVEXITINFO ExitInfo;
12714	RT_ZERO(ExitInfo);
12715	ExitInfo.uReason = uExitReason;
12716	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12717	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12718	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12719	}
12720	else
12721	rcStrict = hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
12722	break;
12723	}
12724
12725	case VMX_EXIT_INVPCID:
12726	{
12727	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
12728	{
12729	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12730	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12731	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12732	AssertRCReturn(rc, rc);
12733
12734	VMXVEXITINFO ExitInfo;
12735	RT_ZERO(ExitInfo);
12736	ExitInfo.uReason = uExitReason;
12737	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12738	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12739	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12740	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12741	}
12742	else
12743	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12744	break;
12745	}
12746
12747	case VMX_EXIT_RDMSR:
12748	{
12749	uint32_t fMsrpm;
12750	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12751	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12752	else
12753	fMsrpm = VMXMSRPM_EXIT_RD;
12754
12755	if (fMsrpm & VMXMSRPM_EXIT_RD)
12756	{
12757	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12758	AssertRCReturn(rc, rc);
12759	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12760	}
12761	else
12762	rcStrict = hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
12763	break;
12764	}
12765
12766	case VMX_EXIT_WRMSR:
12767	{
12768	uint32_t fMsrpm;
12769	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
12770	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
12771	else
12772	fMsrpm = VMXMSRPM_EXIT_WR;
12773
12774	if (fMsrpm & VMXMSRPM_EXIT_WR)
12775	{
12776	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12777	AssertRCReturn(rc, rc);
12778	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12779	}
12780	else
12781	rcStrict = hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
12782	break;
12783	}
12784
12785	case VMX_EXIT_TASK_SWITCH:
12786	{
12787	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12788	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12789	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12790	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12791	AssertRCReturn(rc, rc);
12792
12793	VMXVEXITINFO ExitInfo;
12794	RT_ZERO(ExitInfo);
12795	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12796	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12797
12798	VMXVEXITEVENTINFO ExitEventInfo;
12799	RT_ZERO(ExitInfo);
12800	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
12801	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
12802
12803	rcStrict = IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
12804	break;
12805	}
12806
12807	case VMX_EXIT_WBINVD:
12808	{
12809	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
12810	{
12811	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12812	AssertRCReturn(rc, rc);
12813	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12814	}
12815	else
12816	rcStrict = hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
12817	break;
12818	}
12819
12820	case VMX_EXIT_MTF:
12821	{
12822	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
12823	rcStrict = IEMExecVmxVmexit(pVCpu, uExitReason);
12824	break;
12825	}
12826
12827	case VMX_EXIT_APIC_ACCESS:
12828	case VMX_EXIT_XCPT_OR_NMI:
12829	{
12830	/** @todo NSTVMX: APIC-access, Xcpt or NMI, Mov CRx. */
12831	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12832	break;
12833	}
12834
12835	case VMX_EXIT_MOV_CRX:
12836	{
12837	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12838	AssertRCReturn(rc, rc);
12839
12840	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
12841	switch (uAccessType)
12842	{
12843	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
12844	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
12845	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
12846	{
12847	/** @todo NSTVMX: Implement me. */
12848	break;
12849	}
12850
12851	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12852	{
12853	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
12854	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
12855	{
12856	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12857	AssertRCReturn(rc, rc);
12858
12859	RTGCPTR GCPtrEffDst;
12860	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP(pVmxTransient->uExitQual);
12861	if (fMemOperand)
12862	{
12863	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
12864	AssertRCReturn(rc, rc);
12865	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
12866	}
12867	else
12868	GCPtrEffDst = NIL_RTGCPTR;
12869
12870	VMXVEXITINFO ExitInfo;
12871	RT_ZERO(ExitInfo);
12872	ExitInfo.uReason = VMX_EXIT_MOV_CRX;
12873	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12874	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
12875	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12876	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12877	}
12878	else
12879	rcStrict = hmR0VmxExitMovCRx(pVCpu, pVmxTransient);
12880	break;
12881	}
12882
12883	default:
12884	{
12885	pVCpu->hm.s.u32HMError = uAccessType;
12886	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12887	VERR_VMX_UNEXPECTED_EXCEPTION);
12888	}
12889	}
12890	break;
12891	}
12892
12893	case VMX_EXIT_EXT_INT:
12894	{
12895	/* We shouldn't direct physical interrupts to the nested-guest. */
12896	rcStrict = hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12897	break;
12898	}
12899
12900	case VMX_EXIT_INT_WINDOW:
12901	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12902	{
12903	/** @todo NSTVMX: Interrupt window, TPR below threshold. */
12904	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12905	break;
12906	}
12907
12908	case VMX_EXIT_MWAIT:
12909	{
12910	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
12911	{
12912	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12913	AssertRCReturn(rc, rc);
12914	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12915	}
12916	else
12917	rcStrict = hmR0VmxExitMwait(pVCpu, pVmxTransient);
12918	break;
12919	}
12920
12921	case VMX_EXIT_MONITOR:
12922	{
12923	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
12924	{
12925	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12926	AssertRCReturn(rc, rc);
12927	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12928	}
12929	else
12930	rcStrict = hmR0VmxExitMonitor(pVCpu, pVmxTransient);
12931	break;
12932	}
12933
12934	case VMX_EXIT_PAUSE:
12935	{
12936	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
12937	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
12938	* PAUSE when executing a nested-guest? If it does not, we would not need
12939	* to check for the intercepts here. Just call VM-exit... */
12940	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
12941	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
12942	{
12943	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12944	AssertRCReturn(rc, rc);
12945	rcStrict = IEMExecVmxVmexitInstr(pVCpu, uExitReason, pVmxTransient->cbInstr);
12946	}
12947	else
12948	rcStrict = hmR0VmxExitPause(pVCpu, pVmxTransient);
12949	break;
12950	}
12951
12952	case VMX_EXIT_PREEMPT_TIMER:
12953	{
12954	/** @todo NSTVMX: Preempt timer. */
12955	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12956	break;
12957	}
12958
12959	case VMX_EXIT_MOV_DRX:
12960	{
12961	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
12962	{
12963	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12964	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12965	AssertRCReturn(rc, rc);
12966
12967	VMXVEXITINFO ExitInfo;
12968	RT_ZERO(ExitInfo);
12969	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12970	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12971	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12972	}
12973	else
12974	rcStrict = hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
12975	break;
12976	}
12977
12978	case VMX_EXIT_GDTR_IDTR_ACCESS:
12979	case VMX_EXIT_LDTR_TR_ACCESS:
12980	{
12981	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
12982	{
12983	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12984	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12985	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12986	AssertRCReturn(rc, rc);
12987
12988	VMXVEXITINFO ExitInfo;
12989	RT_ZERO(ExitInfo);
12990	ExitInfo.cbInstr = pVmxTransient->cbInstr;
12991	ExitInfo.u64Qual = pVmxTransient->uExitQual;
12992	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
12993	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
12994	}
12995	else
12996	rcStrict = hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient);
12997	break;
12998	}
12999
13000	case VMX_EXIT_RDRAND:
13001	case VMX_EXIT_RDPMC:
13002	case VMX_EXIT_VMREAD:
13003	case VMX_EXIT_VMWRITE:
13004	case VMX_EXIT_RSM:
13005	case VMX_EXIT_RDSEED:
13006	case VMX_EXIT_ENCLS:
13007	case VMX_EXIT_VMFUNC:
13008	case VMX_EXIT_XSAVES:
13009	case VMX_EXIT_XRSTORS:
13010
13011	case VMX_EXIT_TRIPLE_FAULT:
13012	case VMX_EXIT_NMI_WINDOW:
13013	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
13014
13015	case VMX_EXIT_INIT_SIGNAL:
13016	case VMX_EXIT_SIPI:
13017	case VMX_EXIT_IO_SMI:
13018	case VMX_EXIT_SMI:
13019	case VMX_EXIT_ERR_MSR_LOAD:
13020	case VMX_EXIT_ERR_MACHINE_CHECK:
13021	case VMX_EXIT_PML_FULL:
13022	case VMX_EXIT_VIRTUALIZED_EOI:
13023	case VMX_EXIT_APIC_WRITE:
13024	default:
13025	rcStrict = hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13026	break;
13027	}
13028
13029	return rcStrict;
13030	}
13031	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13032
13033
13034	#ifdef VBOX_STRICT
13035	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13036	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13037	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13038
13039	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13040	do { \
13041	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13042	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13043	} while (0)
13044
13045	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13046	do { \
13047	AssertPtr((a_pVCpu)); \
13048	AssertPtr((a_pVmxTransient)); \
13049	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13050	Assert((a_pVmxTransient)->pVmcsInfo); \
13051	Assert(ASMIntAreEnabled()); \
13052	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13053	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13054	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
13055	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13056	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13057	HMVMX_ASSERT_PREEMPT_CPUID(); \
13058	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13059	} while (0)
13060
13061	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13062	do { \
13063	Log4Func(("\n")); \
13064	} while (0)
13065	#else
13066	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13067	do { \
13068	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13069	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13070	} while (0)
13071	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13072	#endif
13073
13074
13075	/**
13076	* Advances the guest RIP by the specified number of bytes.
13077	*
13078	* @param pVCpu The cross context virtual CPU structure.
13079	* @param cbInstr Number of bytes to advance the RIP by.
13080	*
13081	* @remarks No-long-jump zone!!!
13082	*/
13083	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13084	{
13085	/* Advance the RIP. */
13086	pVCpu->cpum.GstCtx.rip += cbInstr;
13087	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13088
13089	/* Update interrupt inhibition. */
13090	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13091	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13092	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13093	}
13094
13095
13096	/**
13097	* Advances the guest RIP after reading it from the VMCS.
13098	*
13099	* @returns VBox status code, no informational status codes.
13100	* @param pVCpu The cross context virtual CPU structure.
13101	* @param pVmxTransient The VMX-transient structure.
13102	*
13103	* @remarks No-long-jump zone!!!
13104	*/
13105	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13106	{
13107	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13108	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13109	AssertRCReturn(rc, rc);
13110
13111	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13112	return VINF_SUCCESS;
13113	}
13114
13115
13116	/**
13117	* Handle a condition that occurred while delivering an event through the guest
13118	* IDT.
13119	*
13120	* @returns Strict VBox status code (i.e. informational status codes too).
13121	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13122	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13123	* to continue execution of the guest which will delivery the \#DF.
13124	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13125	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13126	*
13127	* @param pVCpu The cross context virtual CPU structure.
13128	* @param pVmxTransient The VMX-transient structure.
13129	*
13130	* @remarks No-long-jump zone!!!
13131	*/
13132	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13133	{
13134	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13135
13136	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13137	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13138	AssertRCReturn(rc2, rc2);
13139
13140	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13141	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13142	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13143	{
13144	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13145	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13146
13147	/*
13148	* If the event was a software interrupt (generated with INT n) or a software exception
13149	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13150	* can handle the VM-exit and continue guest execution which will re-execute the
13151	* instruction rather than re-injecting the exception, as that can cause premature
13152	* trips to ring-3 before injection and involve TRPM which currently has no way of
13153	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13154	* the problem).
13155	*/
13156	IEMXCPTRAISE enmRaise;
13157	IEMXCPTRAISEINFO fRaiseInfo;
13158	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13159	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13160	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13161	{
13162	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13163	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13164	}
13165	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13166	{
13167	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13168	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13169	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13170	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13171	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13172	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13173
13174	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13175
13176	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13177	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13178	{
13179	pVmxTransient->fVectoringPF = true;
13180	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13181	}
13182	}
13183	else
13184	{
13185	/*
13186	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13187	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13188	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13189	*/
13190	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13191	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13192	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13193	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13194	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13195	}
13196
13197	/*
13198	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13199	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13200	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13201	* subsequent VM-entry would fail.
13202	*
13203	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13204	*/
13205	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
13206	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13207	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13208	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13209	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13210	{
13211	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
13212	}
13213
13214	switch (enmRaise)
13215	{
13216	case IEMXCPTRAISE_CURRENT_XCPT:
13217	{
13218	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13219	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13220	Assert(rcStrict == VINF_SUCCESS);
13221	break;
13222	}
13223
13224	case IEMXCPTRAISE_PREV_EVENT:
13225	{
13226	uint32_t u32ErrCode;
13227	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13228	{
13229	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13230	AssertRCReturn(rc2, rc2);
13231	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13232	}
13233	else
13234	u32ErrCode = 0;
13235
13236	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13237	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13238	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13239	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13240
13241	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13242	pVCpu->hm.s.Event.u32ErrCode));
13243	Assert(rcStrict == VINF_SUCCESS);
13244	break;
13245	}
13246
13247	case IEMXCPTRAISE_REEXEC_INSTR:
13248	Assert(rcStrict == VINF_SUCCESS);
13249	break;
13250
13251	case IEMXCPTRAISE_DOUBLE_FAULT:
13252	{
13253	/*
13254	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13255	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13256	*/
13257	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13258	{
13259	pVmxTransient->fVectoringDoublePF = true;
13260	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13261	pVCpu->cpum.GstCtx.cr2));
13262	rcStrict = VINF_SUCCESS;
13263	}
13264	else
13265	{
13266	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13267	hmR0VmxSetPendingXcptDF(pVCpu);
13268	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13269	uIdtVector, uExitVector));
13270	rcStrict = VINF_HM_DOUBLE_FAULT;
13271	}
13272	break;
13273	}
13274
13275	case IEMXCPTRAISE_TRIPLE_FAULT:
13276	{
13277	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13278	rcStrict = VINF_EM_RESET;
13279	break;
13280	}
13281
13282	case IEMXCPTRAISE_CPU_HANG:
13283	{
13284	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13285	rcStrict = VERR_EM_GUEST_CPU_HANG;
13286	break;
13287	}
13288
13289	default:
13290	{
13291	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13292	rcStrict = VERR_VMX_IPE_2;
13293	break;
13294	}
13295	}
13296	}
13297	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13298	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13299	&& uExitVector != X86_XCPT_DF
13300	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
13301	{
13302	/*
13303	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13304	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13305	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13306	*/
13307	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
13308	{
13309	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
13310	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
13311	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
13312	}
13313	}
13314
13315	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13316	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13317	return rcStrict;
13318	}
13319
13320
13321	/** @name VM-exit handlers.
13322	* @{
13323	*/
13324	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13325	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13326	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13327
13328	/**
13329	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13330	*/
13331	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13332	{
13333	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13334	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13335	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13336	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13337	return VINF_SUCCESS;
13338	return VINF_EM_RAW_INTERRUPT;
13339	}
13340
13341
13342	/**
13343	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
13344	*/
13345	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13346	{
13347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13348	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13349
13350	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13351	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13352	AssertRCReturn(rc, rc);
13353
13354	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13355	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13356	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13357	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13358
13359	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13360	{
13361	/*
13362	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13363	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13364	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13365	*
13366	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13367	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13368	*/
13369	VMXDispatchHostNmi();
13370	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
13371	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13372	return VINF_SUCCESS;
13373	}
13374
13375	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13376	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13377	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
13378	{ /* likely */ }
13379	else
13380	{
13381	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
13382	rcStrictRc1 = VINF_SUCCESS;
13383	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13384	return rcStrictRc1;
13385	}
13386
13387	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13388	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13389	switch (uIntType)
13390	{
13391	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13392	Assert(uVector == X86_XCPT_DB);
13393	RT_FALL_THRU();
13394	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13395	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13396	RT_FALL_THRU();
13397	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13398	{
13399	/*
13400	* If there's any exception caused as a result of event injection, the resulting
13401	* secondary/final execption will be pending, we shall continue guest execution
13402	* after injecting the event. The page-fault case is complicated and we manually
13403	* handle any currently pending event in hmR0VmxExitXcptPF.
13404	*/
13405	if (!pVCpu->hm.s.Event.fPending)
13406	{ /* likely */ }
13407	else if (uVector != X86_XCPT_PF)
13408	{
13409	rc = VINF_SUCCESS;
13410	break;
13411	}
13412
13413	switch (uVector)
13414	{
13415	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13416	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13417	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13418	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13419	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13420	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13421
13422	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13423	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13424	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13425	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13426	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13427	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13428	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13429	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13430	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13431	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13432	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13433	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13434	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13435	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13436	default:
13437	{
13438	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13439	if (pVmcsInfo->RealMode.fRealOnV86Active)
13440	{
13441	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13442	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13443	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13444
13445	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13446	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13447	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13448	AssertRCReturn(rc, rc);
13449	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13450	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13451	0 /* GCPtrFaultAddress */);
13452	}
13453	else
13454	{
13455	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13456	pVCpu->hm.s.u32HMError = uVector;
13457	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
13458	}
13459	break;
13460	}
13461	}
13462	break;
13463	}
13464
13465	default:
13466	{
13467	pVCpu->hm.s.u32HMError = uExitIntInfo;
13468	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13469	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13470	break;
13471	}
13472	}
13473	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13474	return rc;
13475	}
13476
13477
13478	/**
13479	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13480	*/
13481	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13482	{
13483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13484
13485	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13486	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13487	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13488	AssertRCReturn(rc, rc);
13489
13490	/* Evaluate and deliver pending events and resume guest execution. */
13491	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13492	return VINF_SUCCESS;
13493	}
13494
13495
13496	/**
13497	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13498	*/
13499	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13500	{
13501	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13502
13503	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13504	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13505	{
13506	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13507	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13508	}
13509
13510	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
13511
13512	/*
13513	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13514	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13515	*/
13516	uint32_t fIntrState;
13517	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13518	AssertRCReturn(rc, rc);
13519	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13520	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13521	{
13522	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13523	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13524
13525	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13526	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13527	AssertRCReturn(rc, rc);
13528	}
13529
13530	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13531	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13532	AssertRCReturn(rc, rc);
13533
13534	/* Evaluate and deliver pending events and resume guest execution. */
13535	return VINF_SUCCESS;
13536	}
13537
13538
13539	/**
13540	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13541	*/
13542	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13543	{
13544	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13545	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13546	}
13547
13548
13549	/**
13550	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13551	*/
13552	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13553	{
13554	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13555	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13556	}
13557
13558
13559	/**
13560	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13561	*/
13562	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13563	{
13564	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13565
13566	/*
13567	* Get the state we need and update the exit history entry.
13568	*/
13569	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13570	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13571	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13572	AssertRCReturn(rc, rc);
13573
13574	VBOXSTRICTRC rcStrict;
13575	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13576	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13577	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13578	if (!pExitRec)
13579	{
13580	/*
13581	* Regular CPUID instruction execution.
13582	*/
13583	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13584	if (rcStrict == VINF_SUCCESS)
13585	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13586	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13587	{
13588	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13589	rcStrict = VINF_SUCCESS;
13590	}
13591	}
13592	else
13593	{
13594	/*
13595	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13596	*/
13597	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13598	AssertRCReturn(rc2, rc2);
13599
13600	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13601	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13602
13603	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13604	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13605
13606	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13607	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13608	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13609	}
13610	return rcStrict;
13611	}
13612
13613
13614	/**
13615	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13616	*/
13617	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13618	{
13619	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13620
13621	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13622	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13623	AssertRCReturn(rc, rc);
13624
13625	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13626	return VINF_EM_RAW_EMULATE_INSTR;
13627
13628	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13629	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13630	}
13631
13632
13633	/**
13634	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13635	*/
13636	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13637	{
13638	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13639
13640	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13641	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13642	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13643	AssertRCReturn(rc, rc);
13644
13645	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13646	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13647	{
13648	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13649	we must reset offsetting on VM-entry. See @bugref{6634}. */
13650	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13651	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13652	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13653	}
13654	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13655	{
13656	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13657	rcStrict = VINF_SUCCESS;
13658	}
13659	return rcStrict;
13660	}
13661
13662
13663	/**
13664	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13665	*/
13666	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13667	{
13668	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13669
13670	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13671	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13672	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13673	AssertRCReturn(rc, rc);
13674
13675	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13676	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13677	{
13678	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13679	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13680	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13681	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13682	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13683	}
13684	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13685	{
13686	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13687	rcStrict = VINF_SUCCESS;
13688	}
13689	return rcStrict;
13690	}
13691
13692
13693	/**
13694	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13695	*/
13696	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13697	{
13698	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13699
13700	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13701	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13702	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13703	AssertRCReturn(rc, rc);
13704
13705	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13706	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13707	if (RT_LIKELY(rc == VINF_SUCCESS))
13708	{
13709	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13710	Assert(pVmxTransient->cbInstr == 2);
13711	}
13712	else
13713	{
13714	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13715	rc = VERR_EM_INTERPRETER;
13716	}
13717	return rc;
13718	}
13719
13720
13721	/**
13722	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13723	*/
13724	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13725	{
13726	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13727
13728	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13729	if (EMAreHypercallInstructionsEnabled(pVCpu))
13730	{
13731	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13732	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13733	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13734	AssertRCReturn(rc, rc);
13735
13736	/* Perform the hypercall. */
13737	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13738	if (rcStrict == VINF_SUCCESS)
13739	{
13740	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13741	AssertRCReturn(rc, rc);
13742	}
13743	else
13744	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13745	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13746	\|\| RT_FAILURE(rcStrict));
13747
13748	/* If the hypercall changes anything other than guest's general-purpose registers,
13749	we would need to reload the guest changed bits here before VM-entry. */
13750	}
13751	else
13752	Log4Func(("Hypercalls not enabled\n"));
13753
13754	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13755	if (RT_FAILURE(rcStrict))
13756	{
13757	hmR0VmxSetPendingXcptUD(pVCpu);
13758	rcStrict = VINF_SUCCESS;
13759	}
13760
13761	return rcStrict;
13762	}
13763
13764
13765	/**
13766	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13767	*/
13768	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13769	{
13770	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13771	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13772
13773	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13774	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13775	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13776	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13777	AssertRCReturn(rc, rc);
13778
13779	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13780
13781	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13782	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13783	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13784	{
13785	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13786	rcStrict = VINF_SUCCESS;
13787	}
13788	else
13789	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
13790	VBOXSTRICTRC_VAL(rcStrict)));
13791	return rcStrict;
13792	}
13793
13794
13795	/**
13796	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13797	*/
13798	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13799	{
13800	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13801
13802	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13803	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
13804	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13805	AssertRCReturn(rc, rc);
13806
13807	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
13808	if (rcStrict == VINF_SUCCESS)
13809	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13810	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13811	{
13812	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13813	rcStrict = VINF_SUCCESS;
13814	}
13815
13816	return rcStrict;
13817	}
13818
13819
13820	/**
13821	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13822	*/
13823	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13824	{
13825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13826
13827	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13828	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13829	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13830	AssertRCReturn(rc, rc);
13831
13832	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
13833	if (RT_SUCCESS(rcStrict))
13834	{
13835	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13836	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
13837	rcStrict = VINF_SUCCESS;
13838	}
13839
13840	return rcStrict;
13841	}
13842
13843
13844	/**
13845	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13846	* VM-exit.
13847	*/
13848	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13849	{
13850	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13851	return VINF_EM_RESET;
13852	}
13853
13854
13855	/**
13856	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13857	*/
13858	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13859	{
13860	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13861
13862	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13863	AssertRCReturn(rc, rc);
13864
13865	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13866	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13867	rc = VINF_SUCCESS;
13868	else
13869	rc = VINF_EM_HALT;
13870
13871	if (rc != VINF_SUCCESS)
13872	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13873	return rc;
13874	}
13875
13876
13877	/**
13878	* VM-exit handler for instructions that result in a \#UD exception delivered to
13879	* the guest.
13880	*/
13881	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13882	{
13883	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13884	hmR0VmxSetPendingXcptUD(pVCpu);
13885	return VINF_SUCCESS;
13886	}
13887
13888
13889	/**
13890	* VM-exit handler for expiry of the VMX-preemption timer.
13891	*/
13892	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13893	{
13894	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13895
13896	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13897	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13898
13899	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13900	PVM pVM = pVCpu->CTX_SUFF(pVM);
13901	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13903	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13904	}
13905
13906
13907	/**
13908	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13909	*/
13910	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13911	{
13912	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13913
13914	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13915	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13916	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13917	AssertRCReturn(rc, rc);
13918
13919	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13920	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13921	: HM_CHANGED_RAISED_XCPT_MASK);
13922
13923	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13924	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13925
13926	return rcStrict;
13927	}
13928
13929
13930	/**
13931	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13932	*/
13933	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13934	{
13935	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13936	/** @todo Use VM-exit instruction information. */
13937	return VERR_EM_INTERPRETER;
13938	}
13939
13940
13941	/**
13942	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13943	* Error VM-exit.
13944	*/
13945	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13946	{
13947	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13948	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13949	AssertRCReturn(rc, rc);
13950
13951	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13952	if (RT_FAILURE(rc))
13953	return rc;
13954
13955	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13956	NOREF(uInvalidReason);
13957
13958	#ifdef VBOX_STRICT
13959	uint32_t fIntrState;
13960	RTHCUINTREG uHCReg;
13961	uint64_t u64Val;
13962	uint32_t u32Val;
13963	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13964	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13965	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13966	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13967	AssertRCReturn(rc, rc);
13968
13969	Log4(("uInvalidReason %u\n", uInvalidReason));
13970	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13971	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13972	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13973	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13974
13975	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13976	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13977	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13978	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13979	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13980	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13981	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13982	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13983	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13984	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13985	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13986	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13987
13988	hmR0DumpRegs(pVCpu);
13989	#endif
13990
13991	return VERR_VMX_INVALID_GUEST_STATE;
13992	}
13993
13994	/**
13995	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
13996	*/
13997	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13998	{
13999	/*
14000	* Cummulative notes of all recognized but unexpected VM-exits.
14001	* This does -not- cover those VM-exits like a page-fault occurring when say nested-paging
14002	* is used.
14003	*
14004	* VMX_EXIT_INIT_SIGNAL:
14005	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14006	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14007	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14008	*
14009	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14010	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14011	* See Intel spec. "23.8 Restrictions on VMX operation".
14012	*
14013	* VMX_EXIT_SIPI:
14014	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14015	* activity state is used. We don't make use of it as our guests don't have direct
14016	* access to the host local APIC.
14017	*
14018	* See Intel spec. 25.3 "Other Causes of VM-exits".
14019	*
14020	* VMX_EXIT_IO_SMI:
14021	* VMX_EXIT_SMI:
14022	* This can only happen if we support dual-monitor treatment of SMI, which can be
14023	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14024	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14025	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14026	*
14027	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14028	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14029	*
14030	* VMX_EXIT_ERR_MACHINE_CHECK:
14031	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14032	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14033	* #MC exception abort class exception is raised. We thus cannot assume a
14034	* reasonable chance of continuing any sort of execution and we bail.
14035	*
14036	* See Intel spec. 15.1 "Machine-check Architecture".
14037	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14038	*
14039	* VMX_EXIT_ERR_MSR_LOAD:
14040	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14041	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14042	* execution.
14043	*
14044	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14045	*
14046	* VMX_EXIT_PML_FULL:
14047	* VMX_EXIT_VIRTUALIZED_EOI:
14048	* VMX_EXIT_APIC_WRITE:
14049	* We do not currently support any of these features and thus they are all unexpected
14050	* VM-exits.
14051	*/
14052	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14053	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14054	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14055	}
14056
14057
14058	/**
14059	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
14060	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
14061	* Conditional VM-exit.
14062	*/
14063	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14064	{
14065	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14066
14067	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
14068	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
14069	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14070	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
14071	return VERR_EM_INTERPRETER;
14072	AssertMsgFailed(("Unexpected XDTR access\n"));
14073	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14074	}
14075
14076
14077	/**
14078	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
14079	*/
14080	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14081	{
14082	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14083
14084	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
14085	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14086	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
14087	return VERR_EM_INTERPRETER;
14088	AssertMsgFailed(("Unexpected RDRAND exit\n"));
14089	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14090	}
14091
14092
14093	/**
14094	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14095	*/
14096	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14097	{
14098	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14099
14100	/** @todo Optimize this: We currently drag in in the whole MSR state
14101	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14102	* MSRs required. That would require changes to IEM and possibly CPUM too.
14103	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14104	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14105	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14106	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14107	switch (idMsr)
14108	{
14109	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14110	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14111	}
14112
14113	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14114	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14115	AssertRCReturn(rc, rc);
14116
14117	Log4Func(("ecx=%#RX32\n", idMsr));
14118
14119	#ifdef VBOX_STRICT
14120	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
14121	{
14122	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
14123	&& idMsr != MSR_K6_EFER)
14124	{
14125	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
14126	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14127	}
14128	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14129	{
14130	Assert(pVmcsInfo->pvMsrBitmap);
14131	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14132	if (fMsrpm & VMXMSRPM_ALLOW_RD)
14133	{
14134	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
14135	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14136	}
14137	}
14138	}
14139	#endif
14140
14141	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
14142	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
14143	if (rcStrict == VINF_SUCCESS)
14144	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14145	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
14146	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14147	{
14148	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14149	rcStrict = VINF_SUCCESS;
14150	}
14151	else
14152	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14153
14154	return rcStrict;
14155	}
14156
14157
14158	/**
14159	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
14160	*/
14161	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14162	{
14163	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14164
14165	/** @todo Optimize this: We currently drag in in the whole MSR state
14166	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14167	* MSRs required. That would require changes to IEM and possibly CPUM too.
14168	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
14169	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
14170	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
14171
14172	/*
14173	* The FS and GS base MSRs are not part of the above all-MSRs mask.
14174	* Although we don't need to fetch the base as it will be overwritten shortly, while
14175	* loading guest-state we would also load the entire segment register including limit
14176	* and attributes and thus we need to load them here.
14177	*/
14178	switch (idMsr)
14179	{
14180	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14181	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14182	}
14183
14184	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14185	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14186	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14187	AssertRCReturn(rc, rc);
14188
14189	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14190
14191	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14193
14194	if (rcStrict == VINF_SUCCESS)
14195	{
14196	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14197
14198	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14199	if ( idMsr == MSR_IA32_APICBASE
14200	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14201	&& idMsr <= MSR_IA32_X2APIC_END))
14202	{
14203	/*
14204	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14205	* When full APIC register virtualization is implemented we'll have to make
14206	* sure APIC state is saved from the VMCS before IEM changes it.
14207	*/
14208	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14209	}
14210	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14211	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14212	else if (idMsr == MSR_K6_EFER)
14213	{
14214	/*
14215	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14216	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14217	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14218	*/
14219	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14220	}
14221
14222	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14223	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
14224	{
14225	switch (idMsr)
14226	{
14227	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14228	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14229	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14230	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14231	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14232	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14233	default:
14234	{
14235	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14236	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14237	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14238	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14239	break;
14240	}
14241	}
14242	}
14243	#ifdef VBOX_STRICT
14244	else
14245	{
14246	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14247	switch (idMsr)
14248	{
14249	case MSR_IA32_SYSENTER_CS:
14250	case MSR_IA32_SYSENTER_EIP:
14251	case MSR_IA32_SYSENTER_ESP:
14252	case MSR_K8_FS_BASE:
14253	case MSR_K8_GS_BASE:
14254	{
14255	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14256	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14257	}
14258
14259	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14260	default:
14261	{
14262	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14263	{
14264	/* EFER MSR writes are always intercepted. */
14265	if (idMsr != MSR_K6_EFER)
14266	{
14267	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14268	idMsr));
14269	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14270	}
14271	}
14272
14273	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14274	{
14275	Assert(pVmcsInfo->pvMsrBitmap);
14276	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14277	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14278	{
14279	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14280	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14281	}
14282	}
14283	break;
14284	}
14285	}
14286	}
14287	#endif /* VBOX_STRICT */
14288	}
14289	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14290	{
14291	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14292	rcStrict = VINF_SUCCESS;
14293	}
14294	else
14295	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14296
14297	return rcStrict;
14298	}
14299
14300
14301	/**
14302	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14303	*/
14304	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14305	{
14306	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14307
14308	/** @todo The guest has likely hit a contended spinlock. We might want to
14309	* poke a schedule different guest VCPU. */
14310	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14311	if (RT_SUCCESS(rc))
14312	return VINF_EM_RAW_INTERRUPT;
14313
14314	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14315	return rc;
14316	}
14317
14318
14319	/**
14320	* VM-exit handler for when the TPR value is lowered below the specified
14321	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14322	*/
14323	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14324	{
14325	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14326	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14327
14328	/*
14329	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14330	* We'll re-evaluate pending interrupts and inject them before the next VM
14331	* entry so we can just continue execution here.
14332	*/
14333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14334	return VINF_SUCCESS;
14335	}
14336
14337
14338	/**
14339	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14340	* VM-exit.
14341	*
14342	* @retval VINF_SUCCESS when guest execution can continue.
14343	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14344	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
14345	* interpreter.
14346	*/
14347	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14348	{
14349	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14350	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14351
14352	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14353	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14354	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14355	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14356	AssertRCReturn(rc, rc);
14357
14358	VBOXSTRICTRC rcStrict;
14359	PVM pVM = pVCpu->CTX_SUFF(pVM);
14360	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
14361	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14362	switch (uAccessType)
14363	{
14364	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
14365	{
14366	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14367	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14368	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
14369	AssertMsg( rcStrict == VINF_SUCCESS
14370	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14371	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14372
14373	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14374	{
14375	case 0:
14376	{
14377	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14378	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14380	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14381
14382	/*
14383	* This is a kludge for handling switches back to real mode when we try to use
14384	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14385	* deal with special selector values, so we have to return to ring-3 and run
14386	* there till the selector values are V86 mode compatible.
14387	*
14388	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14389	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
14390	* at the end of this function.
14391	*/
14392	if ( rc == VINF_SUCCESS
14393	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
14394	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14395	&& (uOldCr0 & X86_CR0_PE)
14396	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
14397	{
14398	/** @todo check selectors rather than returning all the time. */
14399	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14400	rcStrict = VINF_EM_RESCHEDULE_REM;
14401	}
14402	break;
14403	}
14404
14405	case 2:
14406	{
14407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14408	/* Nothing to do here, CR2 it's not part of the VMCS. */
14409	break;
14410	}
14411
14412	case 3:
14413	{
14414	Assert( !pVM->hm.s.fNestedPaging
14415	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14416	\|\| pVCpu->hm.s.fUsingDebugLoop);
14417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14418	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14419	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14420	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14421	break;
14422	}
14423
14424	case 4:
14425	{
14426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14427	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14428	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14429	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14430	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14431	break;
14432	}
14433
14434	case 8:
14435	{
14436	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14437	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14438	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14439	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14440	break;
14441	}
14442	default:
14443	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
14444	break;
14445	}
14446	break;
14447	}
14448
14449	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
14450	{
14451	Assert( !pVM->hm.s.fNestedPaging
14452	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14453	\|\| pVCpu->hm.s.fUsingDebugLoop
14454	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
14455	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14456	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
14457	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14458
14459	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
14460	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
14461	AssertMsg( rcStrict == VINF_SUCCESS
14462	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14463	#ifdef VBOX_WITH_STATISTICS
14464	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
14465	{
14466	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14467	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14468	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14469	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14470	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14471	}
14472	#endif
14473	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14474	VBOXSTRICTRC_VAL(rcStrict)));
14475	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14476	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14477	else
14478	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14479	break;
14480	}
14481
14482	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14483	{
14484	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14485	AssertMsg( rcStrict == VINF_SUCCESS
14486	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14487
14488	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14490	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14491	break;
14492	}
14493
14494	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14495	{
14496	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14497	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14498	AssertRCReturn(rc, rc);
14499	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14500	pVmxTransient->uGuestLinearAddr);
14501	AssertMsg( rcStrict == VINF_SUCCESS
14502	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14503	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14504
14505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14507	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14508	break;
14509	}
14510
14511	default:
14512	{
14513	pVCpu->hm.s.u32HMError = uAccessType;
14514	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14515	VERR_VMX_UNEXPECTED_EXCEPTION);
14516	}
14517	}
14518
14519	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14520	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14521	if (rcStrict == VINF_IEM_RAISED_XCPT)
14522	{
14523	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14524	rcStrict = VINF_SUCCESS;
14525	}
14526
14527	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14528	NOREF(pVM);
14529	return rcStrict;
14530	}
14531
14532
14533	/**
14534	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14535	* VM-exit.
14536	*/
14537	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14538	{
14539	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14540	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14541
14542	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14543	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14544	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14545	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14546	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14547	\| CPUMCTX_EXTRN_EFER);
14548	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14549	AssertRCReturn(rc, rc);
14550
14551	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14552	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14553	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14554	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14555	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14556	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14557	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14558	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14559
14560	/*
14561	* Update exit history to see if this exit can be optimized.
14562	*/
14563	VBOXSTRICTRC rcStrict;
14564	PCEMEXITREC pExitRec = NULL;
14565	if ( !fGstStepping
14566	&& !fDbgStepping)
14567	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14568	!fIOString
14569	? !fIOWrite
14570	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14571	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14572	: !fIOWrite
14573	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14574	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14575	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14576	if (!pExitRec)
14577	{
14578	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14579	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14580
14581	uint32_t const cbValue = s_aIOSizes[uIOSize];
14582	uint32_t const cbInstr = pVmxTransient->cbInstr;
14583	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14584	PVM pVM = pVCpu->CTX_SUFF(pVM);
14585	if (fIOString)
14586	{
14587	/*
14588	* INS/OUTS - I/O String instruction.
14589	*
14590	* Use instruction-information if available, otherwise fall back on
14591	* interpreting the instruction.
14592	*/
14593	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14594	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14595	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14596	if (fInsOutsInfo)
14597	{
14598	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14599	AssertRCReturn(rc2, rc2);
14600	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14601	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14602	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14603	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14604	if (fIOWrite)
14605	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14606	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14607	else
14608	{
14609	/*
14610	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14611	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14612	* See Intel Instruction spec. for "INS".
14613	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14614	*/
14615	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14616	}
14617	}
14618	else
14619	rcStrict = IEMExecOne(pVCpu);
14620
14621	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14622	fUpdateRipAlready = true;
14623	}
14624	else
14625	{
14626	/*
14627	* IN/OUT - I/O instruction.
14628	*/
14629	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14630	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14631	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14632	if (fIOWrite)
14633	{
14634	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14635	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14636	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14637	&& !pCtx->eflags.Bits.u1TF)
14638	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14639	}
14640	else
14641	{
14642	uint32_t u32Result = 0;
14643	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14644	if (IOM_SUCCESS(rcStrict))
14645	{
14646	/* Save result of I/O IN instr. in AL/AX/EAX. */
14647	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14648	}
14649	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14650	&& !pCtx->eflags.Bits.u1TF)
14651	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14653	}
14654	}
14655
14656	if (IOM_SUCCESS(rcStrict))
14657	{
14658	if (!fUpdateRipAlready)
14659	{
14660	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14661	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14662	}
14663
14664	/*
14665	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14666	* while booting Fedora 17 64-bit guest.
14667	*
14668	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14669	*/
14670	if (fIOString)
14671	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14672
14673	/*
14674	* If any I/O breakpoints are armed, we need to check if one triggered
14675	* and take appropriate action.
14676	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14677	*/
14678	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14679	AssertRCReturn(rc, rc);
14680
14681	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14682	* execution engines about whether hyper BPs and such are pending. */
14683	uint32_t const uDr7 = pCtx->dr[7];
14684	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14685	&& X86_DR7_ANY_RW_IO(uDr7)
14686	&& (pCtx->cr4 & X86_CR4_DE))
14687	\|\| DBGFBpIsHwIoArmed(pVM)))
14688	{
14689	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14690
14691	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14692	VMMRZCallRing3Disable(pVCpu);
14693	HM_DISABLE_PREEMPT(pVCpu);
14694
14695	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14696
14697	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14698	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14699	{
14700	/* Raise #DB. */
14701	if (fIsGuestDbgActive)
14702	ASMSetDR6(pCtx->dr[6]);
14703	if (pCtx->dr[7] != uDr7)
14704	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14705
14706	hmR0VmxSetPendingXcptDB(pVCpu);
14707	}
14708	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14709	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14710	else if ( rcStrict2 != VINF_SUCCESS
14711	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14712	rcStrict = rcStrict2;
14713	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14714
14715	HM_RESTORE_PREEMPT();
14716	VMMRZCallRing3Enable(pVCpu);
14717	}
14718	}
14719
14720	#ifdef VBOX_STRICT
14721	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14722	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14723	Assert(!fIOWrite);
14724	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14725	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14726	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14727	Assert(fIOWrite);
14728	else
14729	{
14730	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14731	* statuses, that the VMM device and some others may return. See
14732	* IOM_SUCCESS() for guidance. */
14733	AssertMsg( RT_FAILURE(rcStrict)
14734	\|\| rcStrict == VINF_SUCCESS
14735	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14736	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14737	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14738	\|\| rcStrict == VINF_EM_RAW_TO_R3
14739	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14740	# endif
14741	}
14742	#endif
14743	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14744	}
14745	else
14746	{
14747	/*
14748	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14749	*/
14750	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14751	AssertRCReturn(rc2, rc2);
14752	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14753	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14754	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14755	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14756	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14757	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14758
14759	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14760	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14761
14762	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14763	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14764	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14765	}
14766	return rcStrict;
14767	}
14768
14769
14770	/**
14771	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14772	* VM-exit.
14773	*/
14774	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14775	{
14776	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14777
14778	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14779	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14780	AssertRCReturn(rc, rc);
14781	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14782	{
14783	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14784	AssertRCReturn(rc, rc);
14785	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14786	{
14787	uint32_t uErrCode;
14788	RTGCUINTPTR GCPtrFaultAddress;
14789	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14790	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14791	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14792	if (fErrorCodeValid)
14793	{
14794	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14795	AssertRCReturn(rc, rc);
14796	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14797	}
14798	else
14799	uErrCode = 0;
14800
14801	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14802	&& uVector == X86_XCPT_PF)
14803	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14804	else
14805	GCPtrFaultAddress = 0;
14806
14807	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14808	AssertRCReturn(rc, rc);
14809
14810	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14811	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14812
14813	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14814	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14815	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14816	}
14817	}
14818
14819	/* Fall back to the interpreter to emulate the task-switch. */
14820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14821	return VERR_EM_INTERPRETER;
14822	}
14823
14824
14825	/**
14826	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14827	*/
14828	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14829	{
14830	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14831
14832	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14833	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14834	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14835	AssertRCReturn(rc, rc);
14836	return VINF_EM_DBG_STEPPED;
14837	}
14838
14839
14840	/**
14841	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14842	*/
14843	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14844	{
14845	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14847
14848	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14849	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14850	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14851	{
14852	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14853	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14854	{
14855	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14856	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14857	}
14858	}
14859	else
14860	{
14861	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14862	rcStrict1 = VINF_SUCCESS;
14863	return rcStrict1;
14864	}
14865
14866	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14867	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14868	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14869	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14870	AssertRCReturn(rc, rc);
14871
14872	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14873	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14874	VBOXSTRICTRC rcStrict2;
14875	switch (uAccessType)
14876	{
14877	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14878	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14879	{
14880	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14881	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14882	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14883
14884	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14885	GCPhys &= PAGE_BASE_GC_MASK;
14886	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14887	PVM pVM = pVCpu->CTX_SUFF(pVM);
14888	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14889	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14890
14891	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14892	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14893	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14894	CPUMCTX2CORE(pCtx), GCPhys);
14895	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14896	if ( rcStrict2 == VINF_SUCCESS
14897	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14898	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14899	{
14900	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14901	\| HM_CHANGED_GUEST_APIC_TPR);
14902	rcStrict2 = VINF_SUCCESS;
14903	}
14904	break;
14905	}
14906
14907	default:
14908	Log4Func(("uAccessType=%#x\n", uAccessType));
14909	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14910	break;
14911	}
14912
14913	if (rcStrict2 != VINF_SUCCESS)
14914	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14915	return rcStrict2;
14916	}
14917
14918
14919	/**
14920	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14921	* VM-exit.
14922	*/
14923	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14924	{
14925	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14926
14927	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14928	if (pVmxTransient->fWasGuestDebugStateActive)
14929	{
14930	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14931	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14932	}
14933
14934	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14935	if ( !pVCpu->hm.s.fSingleInstruction
14936	&& !pVmxTransient->fWasHyperDebugStateActive)
14937	{
14938	Assert(!DBGFIsStepping(pVCpu));
14939	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14940
14941	/* Don't intercept MOV DRx any more. */
14942	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14943	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14944	AssertRCReturn(rc, rc);
14945
14946	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14947	VMMRZCallRing3Disable(pVCpu);
14948	HM_DISABLE_PREEMPT(pVCpu);
14949
14950	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14951	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14952	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14953
14954	HM_RESTORE_PREEMPT();
14955	VMMRZCallRing3Enable(pVCpu);
14956
14957	#ifdef VBOX_WITH_STATISTICS
14958	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14959	AssertRCReturn(rc, rc);
14960	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14961	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14962	else
14963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14964	#endif
14965	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14966	return VINF_SUCCESS;
14967	}
14968
14969	/*
14970	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14971	* The EFER MSR is always up-to-date.
14972	* Update the segment registers and DR7 from the CPU.
14973	*/
14974	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14975	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14976	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14977	AssertRCReturn(rc, rc);
14978	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14979
14980	PVM pVM = pVCpu->CTX_SUFF(pVM);
14981	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14982	{
14983	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14984	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14985	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14986	if (RT_SUCCESS(rc))
14987	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14988	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14989	}
14990	else
14991	{
14992	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14993	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14994	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14995	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14996	}
14997
14998	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14999	if (RT_SUCCESS(rc))
15000	{
15001	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15002	AssertRCReturn(rc2, rc2);
15003	return VINF_SUCCESS;
15004	}
15005	return rc;
15006	}
15007
15008
15009	/**
15010	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15011	* Conditional VM-exit.
15012	*/
15013	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15014	{
15015	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15016	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15017
15018	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15019	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15020	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15021	{
15022	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
15023	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
15024	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15025	{
15026	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
15027	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15028	}
15029	}
15030	else
15031	{
15032	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15033	rcStrict1 = VINF_SUCCESS;
15034	return rcStrict1;
15035	}
15036
15037	/*
15038	* Get sufficent state and update the exit history entry.
15039	*/
15040	RTGCPHYS GCPhys;
15041	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15042	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15043	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15044	AssertRCReturn(rc, rc);
15045
15046	VBOXSTRICTRC rcStrict;
15047	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15048	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15049	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15050	if (!pExitRec)
15051	{
15052	/*
15053	* If we succeed, resume guest execution.
15054	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15055	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15056	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15057	* weird case. See @bugref{6043}.
15058	*/
15059	PVM pVM = pVCpu->CTX_SUFF(pVM);
15060	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15061	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15062	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15063	if ( rcStrict == VINF_SUCCESS
15064	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15065	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15066	{
15067	/* Successfully handled MMIO operation. */
15068	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15069	\| HM_CHANGED_GUEST_APIC_TPR);
15070	rcStrict = VINF_SUCCESS;
15071	}
15072	}
15073	else
15074	{
15075	/*
15076	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15077	*/
15078	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15079	AssertRCReturn(rc2, rc2);
15080
15081	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15082	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15083
15084	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15085	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15086
15087	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15088	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15089	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15090	}
15091	return VBOXSTRICTRC_TODO(rcStrict);
15092	}
15093
15094
15095	/**
15096	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15097	* VM-exit.
15098	*/
15099	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15100	{
15101	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15102	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15103
15104	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
15105	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15106	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
15107	{
15108	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
15109	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15110	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
15111	}
15112	else
15113	{
15114	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
15115	rcStrict1 = VINF_SUCCESS;
15116	return rcStrict1;
15117	}
15118
15119	RTGCPHYS GCPhys;
15120	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15121	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
15122	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15123	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15124	AssertRCReturn(rc, rc);
15125
15126	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
15127	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
15128
15129	RTGCUINT uErrorCode = 0;
15130	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
15131	uErrorCode \|= X86_TRAP_PF_ID;
15132	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
15133	uErrorCode \|= X86_TRAP_PF_RW;
15134	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
15135	uErrorCode \|= X86_TRAP_PF_P;
15136
15137	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
15138
15139
15140	/* Handle the pagefault trap for the nested shadow table. */
15141	PVM pVM = pVCpu->CTX_SUFF(pVM);
15142	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15143
15144	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
15145	pCtx->cs.Sel, pCtx->rip));
15146
15147	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
15148	TRPMResetTrap(pVCpu);
15149
15150	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
15151	if ( rcStrict2 == VINF_SUCCESS
15152	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
15153	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
15154	{
15155	/* Successfully synced our nested page tables. */
15156	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
15157	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
15158	return VINF_SUCCESS;
15159	}
15160
15161	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
15162	return rcStrict2;
15163	}
15164
15165	/** @} */
15166
15167	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15168	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
15169	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15170
15171	/**
15172	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
15173	*/
15174	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15175	{
15176	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15177	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
15178
15179	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
15180	AssertRCReturn(rc, rc);
15181
15182	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
15183	{
15184	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
15185	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
15186
15187	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
15188	* provides VM-exit instruction length. If this causes problem later,
15189	* disassemble the instruction like it's done on AMD-V. */
15190	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15191	AssertRCReturn(rc2, rc2);
15192	return rc;
15193	}
15194
15195	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15196	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15197	return rc;
15198	}
15199
15200
15201	/**
15202	* VM-exit exception handler for \#BP (Breakpoint exception).
15203	*/
15204	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15205	{
15206	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15207	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
15208
15209	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15210	AssertRCReturn(rc, rc);
15211
15212	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15213	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15214	if (rc == VINF_EM_RAW_GUEST_TRAP)
15215	{
15216	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15217	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15218	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15219	AssertRCReturn(rc, rc);
15220
15221	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15222	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15223	}
15224
15225	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15226	return rc;
15227	}
15228
15229
15230	/**
15231	* VM-exit exception handler for \#AC (alignment check exception).
15232	*/
15233	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15234	{
15235	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15236
15237	/*
15238	* Re-inject it. We'll detect any nesting before getting here.
15239	*/
15240	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15241	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15242	AssertRCReturn(rc, rc);
15243	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15244
15245	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15246	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15247	return VINF_SUCCESS;
15248	}
15249
15250
15251	/**
15252	* VM-exit exception handler for \#DB (Debug exception).
15253	*/
15254	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15255	{
15256	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15257	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15258
15259	/*
15260	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
15261	* for processing.
15262	*/
15263	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15264
15265	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15266	uint64_t const uDR6 = X86_DR6_INIT_VAL
15267	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15268	\| X86_DR6_BD \| X86_DR6_BS));
15269
15270	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15271	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15272	Log6Func(("rc=%Rrc\n", rc));
15273	if (rc == VINF_EM_RAW_GUEST_TRAP)
15274	{
15275	/*
15276	* The exception was for the guest. Update DR6, DR7.GD and
15277	* IA32_DEBUGCTL.LBR before forwarding it.
15278	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15279	*/
15280	VMMRZCallRing3Disable(pVCpu);
15281	HM_DISABLE_PREEMPT(pVCpu);
15282
15283	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15284	pCtx->dr[6] \|= uDR6;
15285	if (CPUMIsGuestDebugStateActive(pVCpu))
15286	ASMSetDR6(pCtx->dr[6]);
15287
15288	HM_RESTORE_PREEMPT();
15289	VMMRZCallRing3Enable(pVCpu);
15290
15291	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15292	AssertRCReturn(rc, rc);
15293
15294	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15295	pCtx->dr[7] &= ~X86_DR7_GD;
15296
15297	/* Paranoia. */
15298	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15299	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15300
15301	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15302	AssertRCReturn(rc, rc);
15303
15304	/*
15305	* Raise #DB in the guest.
15306	*
15307	* It is important to reflect exactly what the VM-exit gave us (preserving the
15308	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15309	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15310	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15311	*
15312	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15313	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15314	*/
15315	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15316	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15317	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15318	AssertRCReturn(rc, rc);
15319	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15320	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15321	return VINF_SUCCESS;
15322	}
15323
15324	/*
15325	* Not a guest trap, must be a hypervisor related debug event then.
15326	* Update DR6 in case someone is interested in it.
15327	*/
15328	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15329	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15330	CPUMSetHyperDR6(pVCpu, uDR6);
15331
15332	return rc;
15333	}
15334
15335
15336	/**
15337	* Hacks its way around the lovely mesa driver's backdoor accesses.
15338	*
15339	* @sa hmR0SvmHandleMesaDrvGp.
15340	*/
15341	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15342	{
15343	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15344	RT_NOREF(pCtx);
15345
15346	/* For now we'll just skip the instruction. */
15347	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15348	}
15349
15350
15351	/**
15352	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15353	* backdoor logging w/o checking what it is running inside.
15354	*
15355	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15356	* backdoor port and magic numbers loaded in registers.
15357	*
15358	* @returns true if it is, false if it isn't.
15359	* @sa hmR0SvmIsMesaDrvGp.
15360	*/
15361	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15362	{
15363	/* 0xed: IN eAX,dx */
15364	uint8_t abInstr[1];
15365	if (pVmxTransient->cbInstr != sizeof(abInstr))
15366	return false;
15367
15368	/* Check that it is #GP(0). */
15369	if (pVmxTransient->uExitIntErrorCode != 0)
15370	return false;
15371
15372	/* Check magic and port. */
15373	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15374	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15375	if (pCtx->rax != UINT32_C(0x564d5868))
15376	return false;
15377	if (pCtx->dx != UINT32_C(0x5658))
15378	return false;
15379
15380	/* Flat ring-3 CS. */
15381	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15382	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15383	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15384	if (pCtx->cs.Attr.n.u2Dpl != 3)
15385	return false;
15386	if (pCtx->cs.u64Base != 0)
15387	return false;
15388
15389	/* Check opcode. */
15390	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15391	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15392	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15393	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15394	if (RT_FAILURE(rc))
15395	return false;
15396	if (abInstr[0] != 0xed)
15397	return false;
15398
15399	return true;
15400	}
15401
15402
15403	/**
15404	* VM-exit exception handler for \#GP (General-protection exception).
15405	*
15406	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15407	*/
15408	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15409	{
15410	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15412
15413	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15414	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15415	if (pVmcsInfo->RealMode.fRealOnV86Active)
15416	{ /* likely */ }
15417	else
15418	{
15419	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15420	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
15421	#endif
15422	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15423	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15424	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15425	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15426	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15427	AssertRCReturn(rc, rc);
15428	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15429	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15430
15431	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15432	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15433	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15434	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15435	else
15436	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15437	return rc;
15438	}
15439
15440	Assert(CPUMIsGuestInRealModeEx(pCtx));
15441	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15442
15443	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15444	AssertRCReturn(rc, rc);
15445
15446	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15447	if (rcStrict == VINF_SUCCESS)
15448	{
15449	if (!CPUMIsGuestInRealModeEx(pCtx))
15450	{
15451	/*
15452	* The guest is no longer in real-mode, check if we can continue executing the
15453	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15454	*/
15455	pVmcsInfo->RealMode.fRealOnV86Active = false;
15456	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15457	{
15458	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15459	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15460	}
15461	else
15462	{
15463	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15464	rcStrict = VINF_EM_RESCHEDULE;
15465	}
15466	}
15467	else
15468	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15469	}
15470	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15471	{
15472	rcStrict = VINF_SUCCESS;
15473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15474	}
15475	return VBOXSTRICTRC_VAL(rcStrict);
15476	}
15477
15478
15479	/**
15480	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15481	* the exception reported in the VMX transient structure back into the VM.
15482	*
15483	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15484	* up-to-date.
15485	*/
15486	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15487	{
15488	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15489	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15490	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15491	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15492	("uVector=%#x u32XcptBitmap=%#X32\n",
15493	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15494	NOREF(pVmcsInfo);
15495	#endif
15496
15497	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15498	hmR0VmxCheckExitDueToEventDelivery(). */
15499	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15500	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15501	AssertRCReturn(rc, rc);
15502	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15503
15504	#ifdef DEBUG_ramshankar
15505	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15506	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15507	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15508	#endif
15509
15510	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15511	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15512	return VINF_SUCCESS;
15513	}
15514
15515
15516	/**
15517	* VM-exit exception handler for \#PF (Page-fault exception).
15518	*/
15519	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15520	{
15521	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15522	PVM pVM = pVCpu->CTX_SUFF(pVM);
15523	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15524	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15525	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15526	AssertRCReturn(rc, rc);
15527
15528	if (!pVM->hm.s.fNestedPaging)
15529	{ /* likely */ }
15530	else
15531	{
15532	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15533	Assert(pVCpu->hm.s.fUsingDebugLoop);
15534	#endif
15535	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15536	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15537	{
15538	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15539	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15540	}
15541	else
15542	{
15543	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15544	hmR0VmxSetPendingXcptDF(pVCpu);
15545	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15546	}
15547	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15548	return rc;
15549	}
15550
15551	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15552	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15553	if (pVmxTransient->fVectoringPF)
15554	{
15555	Assert(pVCpu->hm.s.Event.fPending);
15556	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15557	}
15558
15559	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15560	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15561	AssertRCReturn(rc, rc);
15562
15563	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15564	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15565
15566	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15567	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15568
15569	Log4Func(("#PF: rc=%Rrc\n", rc));
15570	if (rc == VINF_SUCCESS)
15571	{
15572	/*
15573	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15574	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15575	*/
15576	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15577	TRPMResetTrap(pVCpu);
15578	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15579	return rc;
15580	}
15581
15582	if (rc == VINF_EM_RAW_GUEST_TRAP)
15583	{
15584	if (!pVmxTransient->fVectoringDoublePF)
15585	{
15586	/* It's a guest page fault and needs to be reflected to the guest. */
15587	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15588	TRPMResetTrap(pVCpu);
15589	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15590	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15591	uGstErrorCode, pVmxTransient->uExitQual);
15592	}
15593	else
15594	{
15595	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15596	TRPMResetTrap(pVCpu);
15597	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15598	hmR0VmxSetPendingXcptDF(pVCpu);
15599	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15600	}
15601
15602	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15603	return VINF_SUCCESS;
15604	}
15605
15606	TRPMResetTrap(pVCpu);
15607	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15608	return rc;
15609	}
15610
15611	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15612	/** @name VMX instruction handlers.
15613	* @{
15614	*/
15615	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15616	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15617	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15618
15619	/**
15620	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15621	*/
15622	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15623	{
15624	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15625
15626	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15627	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15628	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15629	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15630	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15631	AssertRCReturn(rc, rc);
15632
15633	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15634
15635	VMXVEXITINFO ExitInfo;
15636	RT_ZERO(ExitInfo);
15637	ExitInfo.uReason = pVmxTransient->uExitReason;
15638	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15639	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15640	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15641	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15642
15643	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15644	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15645	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15646	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15647	{
15648	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15649	rcStrict = VINF_SUCCESS;
15650	}
15651	return rcStrict;
15652	}
15653
15654
15655	/**
15656	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15657	*/
15658	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15659	{
15660	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15661
15662	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15663	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15664	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15665	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15666	AssertRCReturn(rc, rc);
15667
15668	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15669
15670	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15671	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15672	{
15673	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15674	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15675	}
15676	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15677	return rcStrict;
15678	}
15679
15680
15681	/**
15682	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15683	*/
15684	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15685	{
15686	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15687
15688	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15689	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15690	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15691	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15692	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15693	AssertRCReturn(rc, rc);
15694
15695	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15696
15697	VMXVEXITINFO ExitInfo;
15698	RT_ZERO(ExitInfo);
15699	ExitInfo.uReason = pVmxTransient->uExitReason;
15700	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15701	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15702	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15703	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15704
15705	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15706	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15707	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15708	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15709	{
15710	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15711	rcStrict = VINF_SUCCESS;
15712	}
15713	return rcStrict;
15714	}
15715
15716
15717	/**
15718	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15719	*/
15720	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15721	{
15722	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15723
15724	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15725	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15726	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15727	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15728	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15729	AssertRCReturn(rc, rc);
15730
15731	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15732
15733	VMXVEXITINFO ExitInfo;
15734	RT_ZERO(ExitInfo);
15735	ExitInfo.uReason = pVmxTransient->uExitReason;
15736	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15737	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15738	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15739	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15740
15741	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15742	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15743	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15744	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15745	{
15746	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15747	rcStrict = VINF_SUCCESS;
15748	}
15749	return rcStrict;
15750	}
15751
15752
15753	/**
15754	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15755	*/
15756	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15757	{
15758	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15759
15760	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15761	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15762	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15763	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15764	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15765	AssertRCReturn(rc, rc);
15766
15767	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15768
15769	VMXVEXITINFO ExitInfo;
15770	RT_ZERO(ExitInfo);
15771	ExitInfo.uReason = pVmxTransient->uExitReason;
15772	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15773	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15774	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15775	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15776	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15777
15778	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15779	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15780	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15781	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15782	{
15783	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15784	rcStrict = VINF_SUCCESS;
15785	}
15786	return rcStrict;
15787	}
15788
15789
15790	/**
15791	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15792	*/
15793	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15794	{
15795	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15796
15797	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15798	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15799	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15800	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15801	AssertRCReturn(rc, rc);
15802
15803	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15804
15805	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15806	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15807	{
15808	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15809	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15810	}
15811	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15812	return rcStrict;
15813	}
15814
15815
15816	/**
15817	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15818	*/
15819	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15820	{
15821	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15822
15823	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15824	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15825	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15826	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15827	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15828	AssertRCReturn(rc, rc);
15829
15830	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15831
15832	VMXVEXITINFO ExitInfo;
15833	RT_ZERO(ExitInfo);
15834	ExitInfo.uReason = pVmxTransient->uExitReason;
15835	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15836	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15837	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15838	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15839	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15840
15841	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15842	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15843	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15844	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15845	{
15846	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15847	rcStrict = VINF_SUCCESS;
15848	}
15849	return rcStrict;
15850	}
15851
15852
15853	/**
15854	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15855	*/
15856	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15857	{
15858	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15859
15860	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15861	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15862	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15863	AssertRCReturn(rc, rc);
15864
15865	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15866
15867	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15868	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15869	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15870	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15871	{
15872	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15873	rcStrict = VINF_SUCCESS;
15874	}
15875	return rcStrict;
15876	}
15877
15878
15879	/**
15880	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15881	*/
15882	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15883	{
15884	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15885
15886	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15887	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15888	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15889	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15890	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15891	AssertRCReturn(rc, rc);
15892
15893	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15894
15895	VMXVEXITINFO ExitInfo;
15896	RT_ZERO(ExitInfo);
15897	ExitInfo.uReason = pVmxTransient->uExitReason;
15898	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15899	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15900	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15901	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15902
15903	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15904	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15905	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15906	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15907	{
15908	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15909	rcStrict = VINF_SUCCESS;
15910	}
15911	return rcStrict;
15912	}
15913
15914
15915	/**
15916	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15917	*/
15918	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15919	{
15920	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15921
15922	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15923	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15924	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15925	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15926	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15927	AssertRCReturn(rc, rc);
15928
15929	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15930
15931	VMXVEXITINFO ExitInfo;
15932	RT_ZERO(ExitInfo);
15933	ExitInfo.uReason = pVmxTransient->uExitReason;
15934	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15935	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15936	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15937	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15938
15939	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15940	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15942	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15943	{
15944	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15945	rcStrict = VINF_SUCCESS;
15946	}
15947	return rcStrict;
15948	}
15949
15950	/** @} */
15951	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15952

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78722

Download in other formats: