HMVMXR0.cpp@ 79119

Last change on this file since 79119 was 79106, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Take into account MSR permission merging when verifying permissions in hmR0VmxCheckAutoLoadStoreMsrs.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 698.4 KB

Line
1	/* $Id: HMVMXR0.cpp 79106 2019-06-12 11:18:43Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	#ifdef DEBUG_ramshankar
47	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293	/** Pointer to a const VMX transient state. */
294	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
295
296
297	/**
298	* Memory operand read or write access.
299	*/
300	typedef enum VMXMEMACCESS
301	{
302	VMXMEMACCESS_READ = 0,
303	VMXMEMACCESS_WRITE = 1
304	} VMXMEMACCESS;
305
306	/**
307	* VMX VM-exit handler.
308	*
309	* @returns Strict VBox status code (i.e. informational status codes too).
310	* @param pVCpu The cross context virtual CPU structure.
311	* @param pVmxTransient The VMX-transient structure.
312	*/
313	#ifndef HMVMX_USE_FUNCTION_TABLE
314	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
315	#else
316	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
317	/** Pointer to VM-exit handler. */
318	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
319	#endif
320
321	/**
322	* VMX VM-exit handler, non-strict status code.
323	*
324	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
325	*
326	* @returns VBox status code, no informational status code returned.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pVmxTransient The VMX-transient structure.
329	*
330	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
331	* use of that status code will be replaced with VINF_EM_SOMETHING
332	* later when switching over to IEM.
333	*/
334	#ifndef HMVMX_USE_FUNCTION_TABLE
335	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
336	#else
337	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
338	#endif
339
340
341	/*********************************************************************************************************************************
342	* Internal Functions *
343	*********************************************************************************************************************************/
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
346	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
348	#else
349	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
350	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
351	#endif
352	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
353	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
354	#endif
355
356	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
357	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
358	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
359	#endif
360
361	/** @name VM-exit handlers.
362	* @{
363	*/
364	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
365	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
366	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
369	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
370	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
371	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
372	static FNVMXEXITHANDLER hmR0VmxExitHlt;
373	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
374	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
375	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
376	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
377	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
378	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
379	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
380	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
382	static FNVMXEXITHANDLER hmR0VmxExitVmread;
383	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
384	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
385	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
387	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
391	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
392	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
393	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
394	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
395	static FNVMXEXITHANDLER hmR0VmxExitMwait;
396	static FNVMXEXITHANDLER hmR0VmxExitMtf;
397	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
398	static FNVMXEXITHANDLER hmR0VmxExitPause;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
400	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
401	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
402	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
403	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
404	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
406	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
407	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
411	/** @} */
412
413	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
414	/** @name Nested-guest VM-exit handlers.
415	* @{
416	*/
417	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
418	//static FNVMXEXITHANDLER hmR0VmxExitExtIntNested;
419	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
422	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
423	//static FNVMXEXITHANDLER hmR0VmxExitCpuid;
424	//static FNVMXEXITHANDLER hmR0VmxExitGetsec;
425	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
426	//static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
427	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
428	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
429	//static FNVMXEXITHANDLER hmR0VmxExitVmcall;
430	//static FNVMXEXITHANDLER hmR0VmxExitVmclear;
431	//static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
432	//static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
433	//static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
434	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
435	//static FNVMXEXITHANDLER hmR0VmxExitVmresume;
436	//static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
437	//static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
438	//static FNVMXEXITHANDLER hmR0VmxExitVmxon;
439	//static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
440	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
441	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
442	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
443	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
444	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
445	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
446	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
447	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
448	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
449	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
450	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
451	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
452	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
453	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
454	//static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
455	//static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
456	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
457	//static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
458	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
459	//static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
460	//static FNVMXEXITHANDLER hmR0VmxExitErrUnexpected;
461	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
462	//static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
463	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
464	//static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
465	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
466	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
467	/** @} */
468	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
469
470	/** @name Helpers for hardware exceptions VM-exit handlers.
471	* @{
472	*/
473	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
474	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
475	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
476	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
477	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
478	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
479	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
480	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst);
481	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr);
482	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
483	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg);
484	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu);
485	/** @} */
486
487
488	/*********************************************************************************************************************************
489	* Global Variables *
490	*********************************************************************************************************************************/
491	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
492	static const uint32_t g_aVmcsCacheSegBase[] =
493	{
494	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
495	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
496	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
497	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
498	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
499	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
500	};
501	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
502	#endif
503	static const uint32_t g_aVmcsSegBase[] =
504	{
505	VMX_VMCS_GUEST_ES_BASE,
506	VMX_VMCS_GUEST_CS_BASE,
507	VMX_VMCS_GUEST_SS_BASE,
508	VMX_VMCS_GUEST_DS_BASE,
509	VMX_VMCS_GUEST_FS_BASE,
510	VMX_VMCS_GUEST_GS_BASE
511	};
512	static const uint32_t g_aVmcsSegSel[] =
513	{
514	VMX_VMCS16_GUEST_ES_SEL,
515	VMX_VMCS16_GUEST_CS_SEL,
516	VMX_VMCS16_GUEST_SS_SEL,
517	VMX_VMCS16_GUEST_DS_SEL,
518	VMX_VMCS16_GUEST_FS_SEL,
519	VMX_VMCS16_GUEST_GS_SEL
520	};
521	static const uint32_t g_aVmcsSegLimit[] =
522	{
523	VMX_VMCS32_GUEST_ES_LIMIT,
524	VMX_VMCS32_GUEST_CS_LIMIT,
525	VMX_VMCS32_GUEST_SS_LIMIT,
526	VMX_VMCS32_GUEST_DS_LIMIT,
527	VMX_VMCS32_GUEST_FS_LIMIT,
528	VMX_VMCS32_GUEST_GS_LIMIT
529	};
530	static const uint32_t g_aVmcsSegAttr[] =
531	{
532	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
533	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
534	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
535	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
536	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
537	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
538	};
539	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
540	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
541	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
542	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
543
544	#ifdef HMVMX_USE_FUNCTION_TABLE
545	/**
546	* VMX_EXIT dispatch table.
547	*/
548	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
549	{
550	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
551	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
552	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
553	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
554	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
555	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
556	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
557	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
558	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
559	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
560	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
561	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
562	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
563	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
564	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
565	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
566	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
567	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
568	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
569	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
570	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
571	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
572	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
573	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
574	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
575	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
576	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
577	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
578	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
579	#else
580	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
581	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
582	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
583	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
584	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
585	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
586	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
587	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
588	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
589	#endif
590	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
591	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
592	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
593	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
594	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
595	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
596	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
597	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
598	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
599	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
600	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
601	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
602	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
603	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
604	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
605	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
606	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
607	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
608	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
609	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
610	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
611	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
612	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
613	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
614	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
615	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
616	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
617	#else
618	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
619	#endif
620	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
621	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
622	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
623	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
624	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
625	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
626	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
627	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
628	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
629	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
630	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
631	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
632	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
633	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
634	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
635	};
636	#endif /* HMVMX_USE_FUNCTION_TABLE */
637
638	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
639	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
640	{
641	/* 0 */ "(Not Used)",
642	/* 1 */ "VMCALL executed in VMX root operation.",
643	/* 2 */ "VMCLEAR with invalid physical address.",
644	/* 3 */ "VMCLEAR with VMXON pointer.",
645	/* 4 */ "VMLAUNCH with non-clear VMCS.",
646	/* 5 */ "VMRESUME with non-launched VMCS.",
647	/* 6 */ "VMRESUME after VMXOFF",
648	/* 7 */ "VM-entry with invalid control fields.",
649	/* 8 */ "VM-entry with invalid host state fields.",
650	/* 9 */ "VMPTRLD with invalid physical address.",
651	/* 10 */ "VMPTRLD with VMXON pointer.",
652	/* 11 */ "VMPTRLD with incorrect revision identifier.",
653	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
654	/* 13 */ "VMWRITE to read-only VMCS component.",
655	/* 14 */ "(Not Used)",
656	/* 15 */ "VMXON executed in VMX root operation.",
657	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
658	/* 17 */ "VM-entry with non-launched executing VMCS.",
659	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
660	/* 19 */ "VMCALL with non-clear VMCS.",
661	/* 20 */ "VMCALL with invalid VM-exit control fields.",
662	/* 21 */ "(Not Used)",
663	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
664	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
665	/* 24 */ "VMCALL with invalid SMM-monitor features.",
666	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
667	/* 26 */ "VM-entry with events blocked by MOV SS.",
668	/* 27 */ "(Not Used)",
669	/* 28 */ "Invalid operand to INVEPT/INVVPID."
670	};
671	#endif /* VBOX_STRICT */
672
673
674	/**
675	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
676	*
677	* Any bit set in this mask is owned by the host/hypervisor and would cause a
678	* VM-exit when modified by the guest.
679	*
680	* @returns The static CR0 guest/host mask.
681	* @param pVCpu The cross context virtual CPU structure.
682	*/
683	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
684	{
685	/*
686	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
687	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
688	*/
689	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
690	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
691	* and @bugref{6944}. */
692	PVM pVM = pVCpu->CTX_SUFF(pVM);
693	return ( X86_CR0_PE
694	\| X86_CR0_NE
695	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
696	\| X86_CR0_PG
697	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
698	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
699	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
700	}
701
702
703	/**
704	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
705	*
706	* Any bit set in this mask is owned by the host/hypervisor and would cause a
707	* VM-exit when modified by the guest.
708	*
709	* @returns The static CR4 guest/host mask.
710	* @param pVCpu The cross context virtual CPU structure.
711	*/
712	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
713	{
714	/*
715	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
716	* these bits are reserved on hardware that does not support them. Since the
717	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
718	* these bits and handle it depending on whether we expose them to the guest.
719	*/
720	PVM pVM = pVCpu->CTX_SUFF(pVM);
721	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
722	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
723	return ( X86_CR4_VMXE
724	\| X86_CR4_VME
725	\| X86_CR4_PAE
726	\| X86_CR4_PGE
727	\| X86_CR4_PSE
728	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
729	\| (fPcid ? X86_CR4_PCIDE : 0));
730	}
731
732
733	/**
734	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
735	* area.
736	*
737	* @returns @c true if it's different, @c false otherwise.
738	* @param pVmcsInfo The VMCS info. object.
739	*/
740	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
741	{
742	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
743	&& pVmcsInfo->pvGuestMsrStore);
744	}
745
746
747	/**
748	* Checks whether one of the given Pin-based VM-execution controls are set.
749	*
750	* @returns @c true if set, @c false otherwise.
751	* @param pVCpu The cross context virtual CPU structure.
752	* @param pVmxTransient The VMX-transient structure.
753	* @param uPinCtls The Pin-based VM-execution controls to check.
754	*
755	* @remarks This will not check merged controls when executing a nested-guest
756	* but the original control specified by the guest hypervisor.
757	*/
758	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
759	{
760	if (!pVmxTransient->fIsNestedGuest)
761	{
762	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
763	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
764	}
765	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
766	}
767
768
769	/**
770	* Checks whether one of the given Processor-based VM-execution controls are set.
771	*
772	* @returns @c true if set, @c false otherwise.
773	* @param pVCpu The cross context virtual CPU structure.
774	* @param pVmxTransient The VMX-transient structure.
775	* @param uProcCtls The Processor-based VM-execution controls to check.
776	*
777	* @remarks This will not check merged controls when executing a nested-guest
778	* but the original control specified by the guest hypervisor.
779	*/
780	static bool hmR0VmxIsProcCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
781	{
782	if (!pVmxTransient->fIsNestedGuest)
783	{
784	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
785	return RT_BOOL(pVmcsInfo->u32ProcCtls & uProcCtls);
786	}
787	return CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
788	}
789
790
791	/**
792	* Checks whether one of the given Secondary Processor-based VM-execution controls
793	* are set.
794	*
795	* @returns @c true if set, @c false otherwise.
796	* @param pVCpu The cross context virtual CPU structure.
797	* @param pVmxTransient The VMX-transient structure.
798	* @param uProcCtls2 The Secondary Processor-based VM-execution controls to
799	* check.
800	*
801	* @remarks This will not check merged controls when executing a nested-guest
802	* but the original control specified by the guest hypervisor.
803	*/
804	static bool hmR0VmxIsProcCtls2Set(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls2)
805	{
806	if (!pVmxTransient->fIsNestedGuest)
807	{
808	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
809	return RT_BOOL(pVmcsInfo->u32ProcCtls2 & uProcCtls2);
810	}
811	return CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls2);
812	}
813
814
815	#if 0
816	/**
817	* Checks whether one of the given VM-entry controls are set.
818	*
819	* @returns @c true if set, @c false otherwise.
820	* @param pVCpu The cross context virtual CPU structure.
821	* @param pVmxTransient The VMX-transient structure.
822	* @param uEntryCtls The VM-entry controls to check.
823	*
824	* @remarks This will not check merged controls when executing a nested-guest
825	* but the original control specified by the guest hypervisor.
826	*/
827	static bool hmR0VmxIsEntryCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uEntryCtls)
828	{
829	if (!pVmxTransient->fIsNestedGuest)
830	{
831	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
832	return RT_BOOL(pVmcsInfo->u32EntryCtls & uEntryCtls);
833	}
834	return CPUMIsGuestVmxEntryCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uEntryCtls);
835	}
836
837
838	/**
839	* Checks whether one of the given VM-exit controls are set.
840	*
841	* @returns @c true if set, @c false otherwise.
842	* @param pVCpu The cross context virtual CPU structure.
843	* @param pVmxTransient The VMX-transient structure.
844	* @param uExitCtls The VM-exit controls to check.
845	*
846	* @remarks This will not check merged controls when executing a nested-guest
847	* but the original control specified by the guest hypervisor.
848	*/
849	static bool hmR0VmxIsExitCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uExitCtls)
850	{
851	if (!pVmxTransient->fIsNestedGuest)
852	{
853	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
854	return RT_BOOL(pVmcsInfo->u32ExitCtls & uExitCtls);
855	}
856	return CPUMIsGuestVmxExitCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uExitCtls);
857	}
858	#endif
859
860
861	/**
862	* Adds one or more exceptions to the exception bitmap and commits it to the current
863	* VMCS.
864	*
865	* @returns VBox status code.
866	* @param pVmxTransient The VMX-transient structure.
867	* @param uXcptMask The exception(s) to add.
868	*/
869	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
870	{
871	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
872	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
873	if ((uXcptBitmap & uXcptMask) != uXcptMask)
874	{
875	uXcptBitmap \|= uXcptMask;
876	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
877	AssertRCReturn(rc, rc);
878	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
879	}
880	return VINF_SUCCESS;
881	}
882
883
884	/**
885	* Adds an exception to the exception bitmap and commits it to the current VMCS.
886	*
887	* @returns VBox status code.
888	* @param pVmxTransient The VMX-transient structure.
889	* @param uXcpt The exception to add.
890	*/
891	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
892	{
893	Assert(uXcpt <= X86_XCPT_LAST);
894	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
895	}
896
897
898	/**
899	* Remove one or more exceptions from the exception bitmap and commits it to the
900	* current VMCS.
901	*
902	* This takes care of not removing the exception intercept if a nested-guest
903	* requires the exception to be intercepted.
904	*
905	* @returns VBox status code.
906	* @param pVCpu The cross context virtual CPU structure.
907	* @param pVmxTransient The VMX-transient structure.
908	* @param uXcptMask The exception(s) to remove.
909	*/
910	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
911	{
912	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
913	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
914	if (u32XcptBitmap & uXcptMask)
915	{
916	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
917	if (!pVmxTransient->fIsNestedGuest)
918	{ /* likely */ }
919	else
920	{
921	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
922	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
923	}
924	#endif
925	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
926	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
927	\| RT_BIT(X86_XCPT_DE)
928	\| RT_BIT(X86_XCPT_NM)
929	\| RT_BIT(X86_XCPT_TS)
930	\| RT_BIT(X86_XCPT_UD)
931	\| RT_BIT(X86_XCPT_NP)
932	\| RT_BIT(X86_XCPT_SS)
933	\| RT_BIT(X86_XCPT_GP)
934	\| RT_BIT(X86_XCPT_PF)
935	\| RT_BIT(X86_XCPT_MF));
936	#elif defined(HMVMX_ALWAYS_TRAP_PF)
937	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
938	#endif
939	if (uXcptMask)
940	{
941	/* Validate we are not removing any essential exception intercepts. */
942	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
943	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
944	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
945
946	/* Remove it from the exception bitmap. */
947	u32XcptBitmap &= ~uXcptMask;
948
949	/* Commit and update the cache if necessary. */
950	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
951	{
952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
953	AssertRCReturn(rc, rc);
954	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
955	}
956	}
957	}
958	return VINF_SUCCESS;
959	}
960
961
962	/**
963	* Remove an exceptions from the exception bitmap and commits it to the current
964	* VMCS.
965	*
966	* @returns VBox status code.
967	* @param pVCpu The cross context virtual CPU structure.
968	* @param pVmxTransient The VMX-transient structure.
969	* @param uXcpt The exception to remove.
970	*/
971	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
972	{
973	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
974	}
975
976
977	/**
978	* Loads the VMCS specified by the VMCS info. object.
979	*
980	* @returns VBox status code.
981	* @param pVmcsInfo The VMCS info. object.
982	*/
983	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
984	{
985	Assert(pVmcsInfo);
986	Assert(pVmcsInfo->HCPhysVmcs);
987	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
988
989	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
990	{
991	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
992	if (RT_SUCCESS(rc))
993	{
994	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
995	return VINF_SUCCESS;
996	}
997	return rc;
998	}
999	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
1000	}
1001
1002
1003	/**
1004	* Clears the VMCS specified by the VMCS info. object.
1005	*
1006	* @returns VBox status code.
1007	* @param pVmcsInfo The VMCS info. object.
1008	*/
1009	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1010	{
1011	Assert(pVmcsInfo);
1012	Assert(pVmcsInfo->HCPhysVmcs);
1013	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1014
1015	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1016	if (RT_SUCCESS(rc))
1017	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1018	return rc;
1019	}
1020
1021
1022	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1023	/**
1024	* Switches the current VMCS to the one specified.
1025	*
1026	* @returns VBox status code.
1027	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1028	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1029	*
1030	* @remarks Called with interrupts disabled.
1031	*/
1032	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1033	{
1034	Assert(pVmcsInfoFrom);
1035	Assert(pVmcsInfoTo);
1036
1037	/*
1038	* Clear the VMCS we are switching out if it has not already been cleared.
1039	* This will sync any CPU internal data back to the VMCS.
1040	*/
1041	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1042	{
1043	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1044	if (RT_SUCCESS(rc))
1045	{ /* likely */ }
1046	else
1047	return rc;
1048	}
1049
1050	/*
1051	* Clear the VMCS we are switching to if it has not already been cleared.
1052	* This will initialize the VMCS launch state to "clear" required for loading it.
1053	*
1054	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1055	*/
1056	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1057	{
1058	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1059	if (RT_SUCCESS(rc))
1060	{ /* likely */ }
1061	else
1062	return rc;
1063	}
1064
1065	/*
1066	* Finally, load the VMCS we are switching to.
1067	*/
1068	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1069	}
1070	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1071
1072
1073	/**
1074	* Updates the VM's last error record.
1075	*
1076	* If there was a VMX instruction error, reads the error data from the VMCS and
1077	* updates VCPU's last error record as well.
1078	*
1079	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1080	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1081	* VERR_VMX_INVALID_VMCS_FIELD.
1082	* @param rc The error code.
1083	*/
1084	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1085	{
1086	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1087	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1088	{
1089	AssertPtrReturnVoid(pVCpu);
1090	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1091	}
1092	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1093	}
1094
1095
1096	#ifdef VBOX_STRICT
1097	/**
1098	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1099	* transient structure.
1100	*
1101	* @returns VBox status code.
1102	* @param pVmxTransient The VMX-transient structure.
1103	*
1104	* @remarks No-long-jump zone!!!
1105	*/
1106	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1107	{
1108	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1109	AssertRCReturn(rc, rc);
1110	return VINF_SUCCESS;
1111	}
1112
1113
1114	/**
1115	* Reads the VM-entry exception error code field from the VMCS into
1116	* the VMX transient structure.
1117	*
1118	* @returns VBox status code.
1119	* @param pVmxTransient The VMX-transient structure.
1120	*
1121	* @remarks No-long-jump zone!!!
1122	*/
1123	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1124	{
1125	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1126	AssertRCReturn(rc, rc);
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Reads the VM-entry exception error code field from the VMCS into
1133	* the VMX transient structure.
1134	*
1135	* @returns VBox status code.
1136	* @param pVmxTransient The VMX-transient structure.
1137	*
1138	* @remarks No-long-jump zone!!!
1139	*/
1140	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1141	{
1142	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1143	AssertRCReturn(rc, rc);
1144	return VINF_SUCCESS;
1145	}
1146	#endif /* VBOX_STRICT */
1147
1148
1149	/**
1150	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1151	* transient structure.
1152	*
1153	* @returns VBox status code.
1154	* @param pVmxTransient The VMX-transient structure.
1155	*/
1156	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1157	{
1158	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1159	{
1160	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1161	AssertRCReturn(rc,rc);
1162	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1163	}
1164	return VINF_SUCCESS;
1165	}
1166
1167
1168	/**
1169	* Reads the VM-exit interruption error code from the VMCS into the VMX
1170	* transient structure.
1171	*
1172	* @returns VBox status code.
1173	* @param pVmxTransient The VMX-transient structure.
1174	*/
1175	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1176	{
1177	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1178	{
1179	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1180	AssertRCReturn(rc, rc);
1181	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1182	}
1183	return VINF_SUCCESS;
1184	}
1185
1186
1187	/**
1188	* Reads the VM-exit instruction length field from the VMCS into the VMX
1189	* transient structure.
1190	*
1191	* @returns VBox status code.
1192	* @param pVmxTransient The VMX-transient structure.
1193	*/
1194	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1195	{
1196	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1197	{
1198	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1199	AssertRCReturn(rc, rc);
1200	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1201	}
1202	return VINF_SUCCESS;
1203	}
1204
1205
1206	/**
1207	* Reads the VM-exit instruction-information field from the VMCS into
1208	* the VMX transient structure.
1209	*
1210	* @returns VBox status code.
1211	* @param pVmxTransient The VMX-transient structure.
1212	*/
1213	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1214	{
1215	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1216	{
1217	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1218	AssertRCReturn(rc, rc);
1219	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1220	}
1221	return VINF_SUCCESS;
1222	}
1223
1224
1225	/**
1226	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1227	*
1228	* @returns VBox status code.
1229	* @param pVCpu The cross context virtual CPU structure of the
1230	* calling EMT. (Required for the VMCS cache case.)
1231	* @param pVmxTransient The VMX-transient structure.
1232	*/
1233	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1234	{
1235	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1236	{
1237	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1238	AssertRCReturn(rc, rc);
1239	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1240	}
1241	return VINF_SUCCESS;
1242	}
1243
1244
1245	/**
1246	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1247	*
1248	* @returns VBox status code.
1249	* @param pVCpu The cross context virtual CPU structure of the
1250	* calling EMT. (Required for the VMCS cache case.)
1251	* @param pVmxTransient The VMX-transient structure.
1252	*/
1253	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1254	{
1255	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1256	{
1257	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1258	AssertRCReturn(rc, rc);
1259	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1260	}
1261	return VINF_SUCCESS;
1262	}
1263
1264
1265	/**
1266	* Reads the IDT-vectoring information field from the VMCS into the VMX
1267	* transient structure.
1268	*
1269	* @returns VBox status code.
1270	* @param pVmxTransient The VMX-transient structure.
1271	*
1272	* @remarks No-long-jump zone!!!
1273	*/
1274	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1275	{
1276	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1277	{
1278	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1279	AssertRCReturn(rc, rc);
1280	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1281	}
1282	return VINF_SUCCESS;
1283	}
1284
1285
1286	/**
1287	* Reads the IDT-vectoring error code from the VMCS into the VMX
1288	* transient structure.
1289	*
1290	* @returns VBox status code.
1291	* @param pVmxTransient The VMX-transient structure.
1292	*/
1293	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1294	{
1295	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1296	{
1297	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1298	AssertRCReturn(rc, rc);
1299	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1300	}
1301	return VINF_SUCCESS;
1302	}
1303
1304
1305	/**
1306	* Enters VMX root mode operation on the current CPU.
1307	*
1308	* @returns VBox status code.
1309	* @param pVM The cross context VM structure. Can be
1310	* NULL, after a resume.
1311	* @param HCPhysCpuPage Physical address of the VMXON region.
1312	* @param pvCpuPage Pointer to the VMXON region.
1313	*/
1314	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1315	{
1316	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1317	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1318	Assert(pvCpuPage);
1319	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1320
1321	if (pVM)
1322	{
1323	/* Write the VMCS revision identifier to the VMXON region. */
1324	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1325	}
1326
1327	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1328	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1329
1330	/* Enable the VMX bit in CR4 if necessary. */
1331	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1332
1333	/* Enter VMX root mode. */
1334	int rc = VMXEnable(HCPhysCpuPage);
1335	if (RT_FAILURE(rc))
1336	{
1337	if (!(uOldCr4 & X86_CR4_VMXE))
1338	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1339
1340	if (pVM)
1341	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1342	}
1343
1344	/* Restore interrupts. */
1345	ASMSetFlags(fEFlags);
1346	return rc;
1347	}
1348
1349
1350	/**
1351	* Exits VMX root mode operation on the current CPU.
1352	*
1353	* @returns VBox status code.
1354	*/
1355	static int hmR0VmxLeaveRootMode(void)
1356	{
1357	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1358
1359	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1360	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1361
1362	/* If we're for some reason not in VMX root mode, then don't leave it. */
1363	RTCCUINTREG const uHostCR4 = ASMGetCR4();
1364
1365	int rc;
1366	if (uHostCR4 & X86_CR4_VMXE)
1367	{
1368	/* Exit VMX root mode and clear the VMX bit in CR4. */
1369	VMXDisable();
1370	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1371	rc = VINF_SUCCESS;
1372	}
1373	else
1374	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1375
1376	/* Restore interrupts. */
1377	ASMSetFlags(fEFlags);
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Allocates and maps a physically contiguous page. The allocated page is
1384	* zero'd out (used by various VT-x structures).
1385	*
1386	* @returns IPRT status code.
1387	* @param pMemObj Pointer to the ring-0 memory object.
1388	* @param ppVirt Where to store the virtual address of the
1389	* allocation.
1390	* @param pHCPhys Where to store the physical address of the
1391	* allocation.
1392	*/
1393	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1394	{
1395	AssertPtr(pMemObj);
1396	AssertPtr(ppVirt);
1397	AssertPtr(pHCPhys);
1398	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1399	if (RT_FAILURE(rc))
1400	return rc;
1401	ppVirt = RTR0MemObjAddress(pMemObj);
1402	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1403	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1404	return VINF_SUCCESS;
1405	}
1406
1407
1408	/**
1409	* Frees and unmaps an allocated, physical page.
1410	*
1411	* @param pMemObj Pointer to the ring-0 memory object.
1412	* @param ppVirt Where to re-initialize the virtual address of
1413	* allocation as 0.
1414	* @param pHCPhys Where to re-initialize the physical address of the
1415	* allocation as 0.
1416	*/
1417	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1418	{
1419	AssertPtr(pMemObj);
1420	AssertPtr(ppVirt);
1421	AssertPtr(pHCPhys);
1422	/* NULL is valid, accepted and ignored by the free function below. */
1423	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1424	*pMemObj = NIL_RTR0MEMOBJ;
1425	*ppVirt = NULL;
1426	*pHCPhys = NIL_RTHCPHYS;
1427	}
1428
1429
1430	/**
1431	* Initializes a VMCS info. object.
1432	*
1433	* @param pVmcsInfo The VMCS info. object.
1434	*/
1435	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1436	{
1437	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1438
1439	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1440	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1441	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1442	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1443	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1444	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1445	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1446	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1447	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1448	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1449	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1450	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1451	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1452	}
1453
1454
1455	/**
1456	* Frees the VT-x structures for a VMCS info. object.
1457	*
1458	* @param pVM The cross context VM structure.
1459	* @param pVmcsInfo The VMCS info. object.
1460	*/
1461	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1462	{
1463	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1464
1465	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1466	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1467
1468	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1469	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1470	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1471
1472	hmR0VmxInitVmcsInfo(pVmcsInfo);
1473	}
1474
1475
1476	/**
1477	* Allocates the VT-x structures for a VMCS info. object.
1478	*
1479	* @returns VBox status code.
1480	* @param pVCpu The cross context virtual CPU structure.
1481	* @param pVmcsInfo The VMCS info. object.
1482	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1483	*/
1484	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1485	{
1486	PVM pVM = pVCpu->CTX_SUFF(pVM);
1487
1488	/* Allocate the guest VM control structure (VMCS). */
1489	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1490	if (RT_SUCCESS(rc))
1491	{
1492	if (!fIsNstGstVmcs)
1493	{
1494	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1495	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1496	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1497	{
1498	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1499	NULL /* pR3Ptr /, NULL / pRCPtr */);
1500	}
1501	}
1502	else
1503	{
1504	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1505	Assert(!pVmcsInfo->pbVirtApic);
1506	}
1507
1508	if (RT_SUCCESS(rc))
1509	{
1510	/*
1511	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1512	* transparent accesses of specific MSRs.
1513	*
1514	* If the condition for enabling MSR bitmaps changes here, don't forget to
1515	* update HMIsMsrBitmapActive().
1516	*
1517	* We don't share MSR bitmaps between the guest and nested-guest as we then
1518	* don't need to care about carefully restoring the guest MSR bitmap.
1519	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1520	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1521	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1522	* we do that later while merging VMCS.
1523	*/
1524	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1525	{
1526	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1527	if ( RT_SUCCESS(rc)
1528	&& !fIsNstGstVmcs)
1529	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1530	}
1531
1532	if (RT_SUCCESS(rc))
1533	{
1534	/*
1535	* Allocate the VM-entry MSR-load area for the guest MSRs.
1536	*
1537	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1538	* the guest and nested-guest.
1539	*/
1540	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1541	&pVmcsInfo->HCPhysGuestMsrLoad);
1542	if (RT_SUCCESS(rc))
1543	{
1544	/*
1545	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1546	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1547	*/
1548	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1549	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1550	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1551
1552	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1553	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1554	&pVmcsInfo->HCPhysHostMsrLoad);
1555	}
1556	}
1557	}
1558	}
1559
1560	return rc;
1561	}
1562
1563
1564	/**
1565	* Free all VT-x structures for the VM.
1566	*
1567	* @returns IPRT status code.
1568	* @param pVM The cross context VM structure.
1569	*/
1570	static void hmR0VmxStructsFree(PVM pVM)
1571	{
1572	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1573	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1574	#endif
1575	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1576
1577	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1578	{
1579	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1580	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1581	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1582	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1583	if (pVM->cpum.ro.GuestFeatures.fVmx)
1584	{
1585	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1586	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1587	}
1588	#endif
1589	}
1590	}
1591
1592
1593	/**
1594	* Allocate all VT-x structures for the VM.
1595	*
1596	* @returns IPRT status code.
1597	* @param pVM The cross context VM structure.
1598	*/
1599	static int hmR0VmxStructsAlloc(PVM pVM)
1600	{
1601	/*
1602	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1603	* The VMCS size cannot be more than 4096 bytes.
1604	*
1605	* See Intel spec. Appendix A.1 "Basic VMX Information".
1606	*/
1607	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1608	if (cbVmcs <= X86_PAGE_4K_SIZE)
1609	{ /* likely */ }
1610	else
1611	{
1612	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1613	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1614	}
1615
1616	/*
1617	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1618	*/
1619	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1620	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1621	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1622	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1623	#endif
1624
1625	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1626	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1627	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1628
1629	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1630	{
1631	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1632	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1633	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1634	}
1635
1636	/*
1637	* Allocate per-VM VT-x structures.
1638	*/
1639	int rc = VINF_SUCCESS;
1640	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1641	/* Allocate crash-dump magic scratch page. */
1642	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1643	if (RT_FAILURE(rc))
1644	{
1645	hmR0VmxStructsFree(pVM);
1646	return rc;
1647	}
1648	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1649	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1650	#endif
1651
1652	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1653	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1654	{
1655	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1656	&pVM->hm.s.vmx.HCPhysApicAccess);
1657	if (RT_FAILURE(rc))
1658	{
1659	hmR0VmxStructsFree(pVM);
1660	return rc;
1661	}
1662	}
1663
1664	/*
1665	* Initialize per-VCPU VT-x structures.
1666	*/
1667	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1668	{
1669	/* Allocate the guest VMCS structures. */
1670	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1671	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1672	if (RT_SUCCESS(rc))
1673	{
1674	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1675	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1676	if (pVM->cpum.ro.GuestFeatures.fVmx)
1677	{
1678	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1679	if (RT_SUCCESS(rc))
1680	{ /* likely */ }
1681	else
1682	break;
1683	}
1684	#endif
1685	}
1686	else
1687	break;
1688	}
1689
1690	if (RT_FAILURE(rc))
1691	{
1692	hmR0VmxStructsFree(pVM);
1693	return rc;
1694	}
1695
1696	return VINF_SUCCESS;
1697	}
1698
1699
1700	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1701	/**
1702	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1703	*
1704	* @returns @c true if the MSR is intercepted, @c false otherwise.
1705	* @param pvMsrBitmap The MSR bitmap.
1706	* @param offMsr The MSR byte offset.
1707	* @param iBit The bit offset from the byte offset.
1708	*/
1709	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1710	{
1711	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1712	Assert(pbMsrBitmap);
1713	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1714	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1715	}
1716	#endif
1717
1718
1719	/**
1720	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1721	*
1722	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1723	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1724	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1725	* the read/write access of this MSR.
1726	*
1727	* @param pVCpu The cross context virtual CPU structure.
1728	* @param pVmcsInfo The VMCS info. object.
1729	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1730	* @param idMsr The MSR value.
1731	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1732	* include both a read -and- a write permission!
1733	*
1734	* @sa CPUMGetVmxMsrPermission.
1735	* @remarks Can be called with interrupts disabled.
1736	*/
1737	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1738	{
1739	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1740	Assert(pbMsrBitmap);
1741	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1742
1743	/*
1744	* MSR-bitmap Layout:
1745	* Byte index MSR range Interpreted as
1746	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1747	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1748	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1749	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1750	*
1751	* A bit corresponding to an MSR within the above range causes a VM-exit
1752	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1753	* the MSR range, it always cause a VM-exit.
1754	*
1755	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1756	*/
1757	uint16_t const offBitmapRead = 0;
1758	uint16_t const offBitmapWrite = 0x800;
1759	uint16_t offMsr;
1760	int32_t iBit;
1761	if (idMsr <= UINT32_C(0x00001fff))
1762	{
1763	offMsr = 0;
1764	iBit = idMsr;
1765	}
1766	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1767	{
1768	offMsr = 0x400;
1769	iBit = idMsr - UINT32_C(0xc0000000);
1770	}
1771	else
1772	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1773
1774	/*
1775	* Set the MSR read permission.
1776	*/
1777	uint16_t const offMsrRead = offBitmapRead + offMsr;
1778	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1779	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1780	{
1781	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1782	bool const fClear = !fIsNstGstVmcs ? true
1783	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1784	#else
1785	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1786	bool const fClear = true;
1787	#endif
1788	if (fClear)
1789	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1790	}
1791	else
1792	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1793
1794	/*
1795	* Set the MSR write permission.
1796	*/
1797	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1798	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1799	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1800	{
1801	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1802	bool const fClear = !fIsNstGstVmcs ? true
1803	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1804	#else
1805	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1806	bool const fClear = true;
1807	#endif
1808	if (fClear)
1809	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1810	}
1811	else
1812	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1813	}
1814
1815
1816	/**
1817	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1818	* area.
1819	*
1820	* @returns VBox status code.
1821	* @param pVCpu The cross context virtual CPU structure.
1822	* @param pVmcsInfo The VMCS info. object.
1823	* @param cMsrs The number of MSRs.
1824	*/
1825	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1826	{
1827	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1828	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1829	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
1830	{
1831	/* Commit the MSR counts to the VMCS and update the cache. */
1832	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1833	{
1834	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1835	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1836	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1837	AssertRCReturn(rc, rc);
1838
1839	pVmcsInfo->cEntryMsrLoad = cMsrs;
1840	pVmcsInfo->cExitMsrStore = cMsrs;
1841	pVmcsInfo->cExitMsrLoad = cMsrs;
1842	}
1843	return VINF_SUCCESS;
1844	}
1845
1846	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
1847	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1848	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1849	}
1850
1851
1852	/**
1853	* Adds a new (or updates the value of an existing) guest/host MSR
1854	* pair to be swapped during the world-switch as part of the
1855	* auto-load/store MSR area in the VMCS.
1856	*
1857	* @returns VBox status code.
1858	* @param pVCpu The cross context virtual CPU structure.
1859	* @param pVmxTransient The VMX-transient structure.
1860	* @param idMsr The MSR.
1861	* @param uGuestMsrValue Value of the guest MSR.
1862	* @param fSetReadWrite Whether to set the guest read/write access of this
1863	* MSR (thus not causing a VM-exit).
1864	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1865	* necessary.
1866	*/
1867	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1868	bool fSetReadWrite, bool fUpdateHostMsr)
1869	{
1870	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1871	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1872	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1873	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1874	uint32_t i;
1875
1876	/* Paranoia. */
1877	Assert(pGuestMsrLoad);
1878
1879	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
1880
1881	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1882	for (i = 0; i < cMsrs; i++)
1883	{
1884	if (pGuestMsrLoad[i].u32Msr == idMsr)
1885	break;
1886	}
1887
1888	bool fAdded = false;
1889	if (i == cMsrs)
1890	{
1891	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
1892	++cMsrs;
1893	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1894	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1895
1896	/* Set the guest to read/write this MSR without causing VM-exits. */
1897	if ( fSetReadWrite
1898	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1899	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
1900
1901	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
1902	fAdded = true;
1903	}
1904
1905	/* Update the MSR value for the newly added or already existing MSR. */
1906	pGuestMsrLoad[i].u32Msr = idMsr;
1907	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
1908
1909	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1910	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1911	{
1912	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1913	pGuestMsrStore[i].u32Msr = idMsr;
1914	pGuestMsrStore[i].u64Value = uGuestMsrValue;
1915	}
1916
1917	/* Update the corresponding slot in the host MSR area. */
1918	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1919	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
1920	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
1921	pHostMsr[i].u32Msr = idMsr;
1922
1923	/*
1924	* Only if the caller requests to update the host MSR value AND we've newly added the
1925	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1926	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1927	*
1928	* We do this for performance reasons since reading MSRs may be quite expensive.
1929	*/
1930	if (fAdded)
1931	{
1932	if (fUpdateHostMsr)
1933	{
1934	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1935	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1936	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
1937	}
1938	else
1939	{
1940	/* Someone else can do the work. */
1941	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1942	}
1943	}
1944	return VINF_SUCCESS;
1945	}
1946
1947
1948	/**
1949	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1950	* auto-load/store MSR area in the VMCS.
1951	*
1952	* @returns VBox status code.
1953	* @param pVCpu The cross context virtual CPU structure.
1954	* @param pVmxTransient The VMX-transient structure.
1955	* @param idMsr The MSR.
1956	*/
1957	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1958	{
1959	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1960	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1961	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1962	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1963
1964	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
1965
1966	for (uint32_t i = 0; i < cMsrs; i++)
1967	{
1968	/* Find the MSR. */
1969	if (pGuestMsrLoad[i].u32Msr == idMsr)
1970	{
1971	/*
1972	* If it's the last MSR, we only need to reduce the MSR count.
1973	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
1974	*/
1975	if (i < cMsrs - 1)
1976	{
1977	/* Remove it from the VM-entry MSR-load area. */
1978	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
1979	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
1980
1981	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
1982	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1983	{
1984	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1985	Assert(pGuestMsrStore[i].u32Msr == idMsr);
1986	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
1987	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
1988	}
1989
1990	/* Remove it from the VM-exit MSR-load area. */
1991	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1992	Assert(pHostMsr[i].u32Msr == idMsr);
1993	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
1994	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
1995	}
1996
1997	/* Reduce the count to reflect the removed MSR and bail. */
1998	--cMsrs;
1999	break;
2000	}
2001	}
2002
2003	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2004	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2005	{
2006	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2007	AssertRCReturn(rc, rc);
2008
2009	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2010	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2011	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2012
2013	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2014	return VINF_SUCCESS;
2015	}
2016
2017	return VERR_NOT_FOUND;
2018	}
2019
2020
2021	/**
2022	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2023	*
2024	* @returns @c true if found, @c false otherwise.
2025	* @param pVmcsInfo The VMCS info. object.
2026	* @param idMsr The MSR to find.
2027	*/
2028	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2029	{
2030	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2031	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2032	Assert(pMsrs);
2033	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2034	for (uint32_t i = 0; i < cMsrs; i++)
2035	{
2036	if (pMsrs[i].u32Msr == idMsr)
2037	return true;
2038	}
2039	return false;
2040	}
2041
2042
2043	/**
2044	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2045	*
2046	* @param pVCpu The cross context virtual CPU structure.
2047	* @param pVmcsInfo The VMCS info. object.
2048	*
2049	* @remarks No-long-jump zone!!!
2050	*/
2051	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2052	{
2053	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2054
2055	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2056	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2057	Assert(pHostMsrLoad);
2058	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2059	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2060	for (uint32_t i = 0; i < cMsrs; i++)
2061	{
2062	/*
2063	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2064	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2065	*/
2066	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2067	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2068	else
2069	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2070	}
2071	}
2072
2073
2074	/**
2075	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2076	* perform lazy restoration of the host MSRs while leaving VT-x.
2077	*
2078	* @param pVCpu The cross context virtual CPU structure.
2079	*
2080	* @remarks No-long-jump zone!!!
2081	*/
2082	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2083	{
2084	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2085
2086	/*
2087	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2088	*/
2089	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2090	{
2091	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2092	#if HC_ARCH_BITS == 64
2093	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2094	{
2095	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2096	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2097	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2098	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2099	}
2100	#endif
2101	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2102	}
2103	}
2104
2105
2106	/**
2107	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2108	* lazily while leaving VT-x.
2109	*
2110	* @returns true if it does, false otherwise.
2111	* @param pVCpu The cross context virtual CPU structure.
2112	* @param idMsr The MSR to check.
2113	*/
2114	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2115	{
2116	NOREF(pVCpu);
2117	#if HC_ARCH_BITS == 64
2118	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2119	{
2120	switch (idMsr)
2121	{
2122	case MSR_K8_LSTAR:
2123	case MSR_K6_STAR:
2124	case MSR_K8_SF_MASK:
2125	case MSR_K8_KERNEL_GS_BASE:
2126	return true;
2127	}
2128	}
2129	#else
2130	RT_NOREF(pVCpu, idMsr);
2131	#endif
2132	return false;
2133	}
2134
2135
2136	/**
2137	* Loads a set of guests MSRs to allow read/passthru to the guest.
2138	*
2139	* The name of this function is slightly confusing. This function does NOT
2140	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2141	* common prefix for functions dealing with "lazy restoration" of the shared
2142	* MSRs.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	*
2146	* @remarks No-long-jump zone!!!
2147	*/
2148	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2149	{
2150	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2151	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2152
2153	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2154	#if HC_ARCH_BITS == 64
2155	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2156	{
2157	/*
2158	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2159	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2160	* we can skip a few MSR writes.
2161	*
2162	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2163	* guest MSR values in the guest-CPU context might be different to what's currently
2164	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2165	* CPU, see @bugref{8728}.
2166	*/
2167	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2168	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2169	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2170	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2171	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2172	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2173	{
2174	#ifdef VBOX_STRICT
2175	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2176	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2177	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2178	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2179	#endif
2180	}
2181	else
2182	{
2183	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2184	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2185	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2186	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2187	}
2188	}
2189	#endif
2190	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2191	}
2192
2193
2194	/**
2195	* Performs lazy restoration of the set of host MSRs if they were previously
2196	* loaded with guest MSR values.
2197	*
2198	* @param pVCpu The cross context virtual CPU structure.
2199	*
2200	* @remarks No-long-jump zone!!!
2201	* @remarks The guest MSRs should have been saved back into the guest-CPU
2202	* context by hmR0VmxImportGuestState()!!!
2203	*/
2204	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2205	{
2206	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2207	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2208
2209	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2210	{
2211	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2212	#if HC_ARCH_BITS == 64
2213	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2214	{
2215	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2216	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2217	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2218	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2219	}
2220	#endif
2221	}
2222	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2223	}
2224
2225
2226	/**
2227	* Verifies that our cached values of the VMCS fields are all consistent with
2228	* what's actually present in the VMCS.
2229	*
2230	* @returns VBox status code.
2231	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2232	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2233	* VMCS content. HMCPU error-field is
2234	* updated, see VMX_VCI_XXX.
2235	* @param pVCpu The cross context virtual CPU structure.
2236	* @param pVmcsInfo The VMCS info. object.
2237	*/
2238	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2239	{
2240	uint32_t u32Val;
2241	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2242	AssertRCReturn(rc, rc);
2243	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2244	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2245	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2246	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2247
2248	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2249	AssertRCReturn(rc, rc);
2250	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2251	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2252	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2253	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2254
2255	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2256	AssertRCReturn(rc, rc);
2257	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2258	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2259	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2260	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2261
2262	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2263	AssertRCReturn(rc, rc);
2264	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2265	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2266	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2267	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2268
2269	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2270	{
2271	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2272	AssertRCReturn(rc, rc);
2273	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2274	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2275	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2276	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2277	}
2278
2279	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2280	AssertRCReturn(rc, rc);
2281	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2282	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2283	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2284	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2285
2286	uint64_t u64Val;
2287	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2288	AssertRCReturn(rc, rc);
2289	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2290	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2291	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2292	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2293
2294	return VINF_SUCCESS;
2295	}
2296
2297
2298	#ifdef VBOX_STRICT
2299	/**
2300	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2301	*
2302	* @param pVCpu The cross context virtual CPU structure.
2303	* @param pVmcsInfo The VMCS info. object.
2304	*/
2305	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2306	{
2307	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2308
2309	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2310	{
2311	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2312	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2313	uint64_t uVmcsEferMsrVmcs;
2314	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2315	AssertRC(rc);
2316
2317	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2318	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2319	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2320	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2321	}
2322	}
2323
2324
2325	/**
2326	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2327	* VMCS are correct.
2328	*
2329	* @param pVCpu The cross context virtual CPU structure.
2330	* @param pVmcsInfo The VMCS info. object.
2331	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2332	*/
2333	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2334	{
2335	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2336
2337	/* Read the various MSR-area counts from the VMCS. */
2338	uint32_t cEntryLoadMsrs;
2339	uint32_t cExitStoreMsrs;
2340	uint32_t cExitLoadMsrs;
2341	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2342	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2343	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2344
2345	/* Verify all the MSR counts are the same. */
2346	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2347	Assert(cExitStoreMsrs == cExitLoadMsrs);
2348	uint32_t const cMsrs = cExitLoadMsrs;
2349
2350	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2351	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2352
2353	/* Verify the MSR counts are within the allocated page size. */
2354	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2355
2356	/* Verify the relevant contents of the MSR areas match. */
2357	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2358	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2359	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2360	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2361	for (uint32_t i = 0; i < cMsrs; i++)
2362	{
2363	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2364	if (fSeparateExitMsrStorePage)
2365	{
2366	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2367	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2368	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2369	}
2370
2371	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2372	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2373	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2374
2375	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2376	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2377	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2378	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2379
2380	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2381	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2382	if (fIsEferMsr)
2383	{
2384	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2385	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2386	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2387	}
2388
2389	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2390	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2391	{
2392	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2393	if (fIsEferMsr)
2394	{
2395	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2396	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2397	}
2398	else
2399	{
2400	if (!fIsNstGstVmcs)
2401	{
2402	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2403	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2404	}
2405	else
2406	{
2407	/*
2408	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2409	* execute a nested-guest with MSR passthrough.
2410	*
2411	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2412	* allow passthrough too.
2413	*/
2414	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2415	Assert(pvMsrBitmapNstGst);
2416	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2417	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2418	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2419	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2420	}
2421	}
2422	}
2423
2424	/* Move to the next MSR. */
2425	pHostMsrLoad++;
2426	pGuestMsrLoad++;
2427	pGuestMsrStore++;
2428	}
2429	}
2430	#endif /* VBOX_STRICT */
2431
2432
2433	/**
2434	* Flushes the TLB using EPT.
2435	*
2436	* @returns VBox status code.
2437	* @param pVCpu The cross context virtual CPU structure of the calling
2438	* EMT. Can be NULL depending on @a enmTlbFlush.
2439	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2440	* enmTlbFlush.
2441	* @param enmTlbFlush Type of flush.
2442	*
2443	* @remarks Caller is responsible for making sure this function is called only
2444	* when NestedPaging is supported and providing @a enmTlbFlush that is
2445	* supported by the CPU.
2446	* @remarks Can be called with interrupts disabled.
2447	*/
2448	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2449	{
2450	uint64_t au64Descriptor[2];
2451	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2452	au64Descriptor[0] = 0;
2453	else
2454	{
2455	Assert(pVCpu);
2456	Assert(pVmcsInfo);
2457	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2458	}
2459	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2460
2461	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2462	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2463
2464	if ( RT_SUCCESS(rc)
2465	&& pVCpu)
2466	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2467	}
2468
2469
2470	/**
2471	* Flushes the TLB using VPID.
2472	*
2473	* @returns VBox status code.
2474	* @param pVCpu The cross context virtual CPU structure of the calling
2475	* EMT. Can be NULL depending on @a enmTlbFlush.
2476	* @param enmTlbFlush Type of flush.
2477	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2478	* on @a enmTlbFlush).
2479	*
2480	* @remarks Can be called with interrupts disabled.
2481	*/
2482	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2483	{
2484	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2485
2486	uint64_t au64Descriptor[2];
2487	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2488	{
2489	au64Descriptor[0] = 0;
2490	au64Descriptor[1] = 0;
2491	}
2492	else
2493	{
2494	AssertPtr(pVCpu);
2495	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2496	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2497	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2498	au64Descriptor[1] = GCPtr;
2499	}
2500
2501	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2502	AssertMsg(rc == VINF_SUCCESS,
2503	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2504
2505	if ( RT_SUCCESS(rc)
2506	&& pVCpu)
2507	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2508	NOREF(rc);
2509	}
2510
2511
2512	/**
2513	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2514	* otherwise there is nothing really to invalidate.
2515	*
2516	* @returns VBox status code.
2517	* @param pVCpu The cross context virtual CPU structure.
2518	* @param GCVirt Guest virtual address of the page to invalidate.
2519	*/
2520	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2521	{
2522	AssertPtr(pVCpu);
2523	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2524
2525	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2526	{
2527	/*
2528	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2529	* the EPT case. See @bugref{6043} and @bugref{6177}.
2530	*
2531	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2532	* as this function maybe called in a loop with individual addresses.
2533	*/
2534	PVM pVM = pVCpu->CTX_SUFF(pVM);
2535	if (pVM->hm.s.vmx.fVpid)
2536	{
2537	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2538
2539	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2540	/*
2541	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2542	* where executing INVVPID outside 64-bit mode does not flush translations of
2543	* 64-bit linear addresses, see @bugref{6208#c72}.
2544	*/
2545	if (RT_HI_U32(GCVirt))
2546	fVpidFlush = false;
2547	#endif
2548
2549	if (fVpidFlush)
2550	{
2551	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2552	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2553	}
2554	else
2555	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2556	}
2557	else if (pVM->hm.s.fNestedPaging)
2558	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2559	}
2560
2561	return VINF_SUCCESS;
2562	}
2563
2564
2565	/**
2566	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2567	* case where neither EPT nor VPID is supported by the CPU.
2568	*
2569	* @param pHostCpu The HM physical-CPU structure.
2570	* @param pVCpu The cross context virtual CPU structure.
2571	*
2572	* @remarks Called with interrupts disabled.
2573	*/
2574	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2575	{
2576	AssertPtr(pVCpu);
2577	AssertPtr(pHostCpu);
2578
2579	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2580
2581	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2582	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2583	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2584	pVCpu->hm.s.fForceTLBFlush = false;
2585	return;
2586	}
2587
2588
2589	/**
2590	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2591	*
2592	* @param pHostCpu The HM physical-CPU structure.
2593	* @param pVCpu The cross context virtual CPU structure.
2594	* @param pVmcsInfo The VMCS info. object.
2595	*
2596	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2597	* nomenclature. The reason is, to avoid confusion in compare statements
2598	* since the host-CPU copies are named "ASID".
2599	*
2600	* @remarks Called with interrupts disabled.
2601	*/
2602	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2603	{
2604	#ifdef VBOX_WITH_STATISTICS
2605	bool fTlbFlushed = false;
2606	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2607	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2608	if (!fTlbFlushed) \
2609	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2610	} while (0)
2611	#else
2612	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2613	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2614	#endif
2615
2616	AssertPtr(pVCpu);
2617	AssertPtr(pHostCpu);
2618	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2619
2620	PVM pVM = pVCpu->CTX_SUFF(pVM);
2621	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2622	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2623	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2624
2625	/*
2626	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2627	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2628	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2629	* cannot reuse the current ASID anymore.
2630	*/
2631	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2632	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2633	{
2634	++pHostCpu->uCurrentAsid;
2635	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2636	{
2637	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2638	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2639	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2640	}
2641
2642	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2643	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2644	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2645
2646	/*
2647	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2648	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2649	*/
2650	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2651	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2652	HMVMX_SET_TAGGED_TLB_FLUSHED();
2653	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2654	}
2655	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2656	{
2657	/*
2658	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2659	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2660	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2661	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2662	* mappings, see @bugref{6568}.
2663	*
2664	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2665	*/
2666	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2667	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2668	HMVMX_SET_TAGGED_TLB_FLUSHED();
2669	}
2670
2671	pVCpu->hm.s.fForceTLBFlush = false;
2672	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2673
2674	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2675	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2676	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2677	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2678	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2679	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2680	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2681	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2682	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2683
2684	/* Update VMCS with the VPID. */
2685	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2686	AssertRC(rc);
2687
2688	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2689	}
2690
2691
2692	/**
2693	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2694	*
2695	* @param pHostCpu The HM physical-CPU structure.
2696	* @param pVCpu The cross context virtual CPU structure.
2697	* @param pVmcsInfo The VMCS info. object.
2698	*
2699	* @remarks Called with interrupts disabled.
2700	*/
2701	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2702	{
2703	AssertPtr(pVCpu);
2704	AssertPtr(pHostCpu);
2705	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2706	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2707	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2708
2709	/*
2710	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2711	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2712	*/
2713	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2714	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2715	{
2716	pVCpu->hm.s.fForceTLBFlush = true;
2717	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2718	}
2719
2720	/* Check for explicit TLB flushes. */
2721	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2722	{
2723	pVCpu->hm.s.fForceTLBFlush = true;
2724	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2725	}
2726
2727	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2728	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2729
2730	if (pVCpu->hm.s.fForceTLBFlush)
2731	{
2732	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2733	pVCpu->hm.s.fForceTLBFlush = false;
2734	}
2735	}
2736
2737
2738	/**
2739	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2740	*
2741	* @param pHostCpu The HM physical-CPU structure.
2742	* @param pVCpu The cross context virtual CPU structure.
2743	*
2744	* @remarks Called with interrupts disabled.
2745	*/
2746	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2747	{
2748	AssertPtr(pVCpu);
2749	AssertPtr(pHostCpu);
2750	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2751	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2752	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2753
2754	/*
2755	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2756	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2757	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2758	* cannot reuse the current ASID anymore.
2759	*/
2760	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2761	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2762	{
2763	pVCpu->hm.s.fForceTLBFlush = true;
2764	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2765	}
2766
2767	/* Check for explicit TLB flushes. */
2768	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2769	{
2770	/*
2771	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2772	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2773	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2774	* include fExplicitFlush's too) - an obscure corner case.
2775	*/
2776	pVCpu->hm.s.fForceTLBFlush = true;
2777	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2778	}
2779
2780	PVM pVM = pVCpu->CTX_SUFF(pVM);
2781	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2782	if (pVCpu->hm.s.fForceTLBFlush)
2783	{
2784	++pHostCpu->uCurrentAsid;
2785	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2786	{
2787	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2788	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2789	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2790	}
2791
2792	pVCpu->hm.s.fForceTLBFlush = false;
2793	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2794	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2795	if (pHostCpu->fFlushAsidBeforeUse)
2796	{
2797	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2798	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2799	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2800	{
2801	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2802	pHostCpu->fFlushAsidBeforeUse = false;
2803	}
2804	else
2805	{
2806	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2807	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2808	}
2809	}
2810	}
2811
2812	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2813	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2814	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2815	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2816	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2817	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2818	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2819
2820	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2821	AssertRC(rc);
2822	}
2823
2824
2825	/**
2826	* Flushes the guest TLB entry based on CPU capabilities.
2827	*
2828	* @param pHostCpu The HM physical-CPU structure.
2829	* @param pVCpu The cross context virtual CPU structure.
2830	* @param pVmcsInfo The VMCS info. object.
2831	*
2832	* @remarks Called with interrupts disabled.
2833	*/
2834	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2835	{
2836	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2837	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2838	#endif
2839	PVM pVM = pVCpu->CTX_SUFF(pVM);
2840	switch (pVM->hm.s.vmx.enmTlbFlushType)
2841	{
2842	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2843	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2844	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2845	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2846	default:
2847	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2848	break;
2849	}
2850	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2851	}
2852
2853
2854	/**
2855	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2856	* TLB entries from the host TLB before VM-entry.
2857	*
2858	* @returns VBox status code.
2859	* @param pVM The cross context VM structure.
2860	*/
2861	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2862	{
2863	/*
2864	* Determine optimal flush type for nested paging.
2865	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2866	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2867	*/
2868	if (pVM->hm.s.fNestedPaging)
2869	{
2870	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2871	{
2872	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2873	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2874	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2875	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2876	else
2877	{
2878	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2879	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2880	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2881	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2882	}
2883
2884	/* Make sure the write-back cacheable memory type for EPT is supported. */
2885	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2886	{
2887	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2888	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2889	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2890	}
2891
2892	/* EPT requires a page-walk length of 4. */
2893	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2894	{
2895	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2896	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2897	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2898	}
2899	}
2900	else
2901	{
2902	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2903	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2904	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2905	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2906	}
2907	}
2908
2909	/*
2910	* Determine optimal flush type for VPID.
2911	*/
2912	if (pVM->hm.s.vmx.fVpid)
2913	{
2914	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2915	{
2916	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2917	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2918	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2919	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2920	else
2921	{
2922	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2923	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2924	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2925	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2926	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2927	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2928	pVM->hm.s.vmx.fVpid = false;
2929	}
2930	}
2931	else
2932	{
2933	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2934	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2935	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2936	pVM->hm.s.vmx.fVpid = false;
2937	}
2938	}
2939
2940	/*
2941	* Setup the handler for flushing tagged-TLBs.
2942	*/
2943	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2944	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2945	else if (pVM->hm.s.fNestedPaging)
2946	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2947	else if (pVM->hm.s.vmx.fVpid)
2948	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2949	else
2950	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2951	return VINF_SUCCESS;
2952	}
2953
2954
2955	/**
2956	* Sets up the virtual-APIC page address for the VMCS.
2957	*
2958	* @returns VBox status code.
2959	* @param pVCpu The cross context virtual CPU structure.
2960	* @param pVmcsInfo The VMCS info. object.
2961	*/
2962	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2963	{
2964	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2965	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2966	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2967	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2968	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2969	}
2970
2971
2972	/**
2973	* Sets up the MSR-bitmap address for the VMCS.
2974	*
2975	* @returns VBox status code.
2976	* @param pVCpu The cross context virtual CPU structure.
2977	* @param pVmcsInfo The VMCS info. object.
2978	*/
2979	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2980	{
2981	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
2982	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2983	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2984	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2985	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2986	}
2987
2988
2989	/**
2990	* Sets up the APIC-access page address for the VMCS.
2991	*
2992	* @returns VBox status code.
2993	* @param pVCpu The cross context virtual CPU structure.
2994	*/
2995	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2996	{
2997	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2998	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2999	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3000	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3001	}
3002
3003
3004	/**
3005	* Sets up the VMCS link pointer for the VMCS.
3006	*
3007	* @returns VBox status code.
3008	* @param pVCpu The cross context virtual CPU structure.
3009	* @param pVmcsInfo The VMCS info. object.
3010	*/
3011	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3012	{
3013	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3014	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
3015	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
3016	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
3017	}
3018
3019
3020	/**
3021	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3022	* in the VMCS.
3023	*
3024	* @returns VBox status code.
3025	* @param pVCpu The cross context virtual CPU structure.
3026	* @param pVmcsInfo The VMCS info. object.
3027	*/
3028	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3029	{
3030	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3031
3032	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3033	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3034	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3035
3036	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3037	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3038	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3039
3040	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3041	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3042	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3043
3044	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3045	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3046	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3047	AssertRCReturn(rc, rc);
3048	return VINF_SUCCESS;
3049	}
3050
3051
3052	/**
3053	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3054	*
3055	* @param pVCpu The cross context virtual CPU structure.
3056	* @param pVmcsInfo The VMCS info. object.
3057	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3058	*/
3059	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3060	{
3061	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3062
3063	/*
3064	* The guest can access the following MSRs (read, write) without causing
3065	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3066	*/
3067	PVM pVM = pVCpu->CTX_SUFF(pVM);
3068	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3069	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3070	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3071	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3072	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3073
3074	/*
3075	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3076	* associated with then. We never need to intercept access (writes need to be
3077	* executed without causing a VM-exit, reads will #GP fault anyway).
3078	*
3079	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3080	* read/write them. We swap the the guest/host MSR value using the
3081	* auto-load/store MSR area.
3082	*/
3083	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3084	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3085	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3086	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3087	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3088	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3089
3090	#if HC_ARCH_BITS == 64
3091	/*
3092	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3093	* required for 64-bit guests.
3094	*/
3095	if (pVM->hm.s.fAllow64BitGuests)
3096	{
3097	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3098	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3099	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3100	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3101	}
3102	#endif
3103
3104	/*
3105	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3106	*/
3107	#ifdef VBOX_STRICT
3108	Assert(pVmcsInfo->pvMsrBitmap);
3109	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3110	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3111	#endif
3112	}
3113
3114
3115	/**
3116	* Sets up pin-based VM-execution controls in the VMCS.
3117	*
3118	* @returns VBox status code.
3119	* @param pVCpu The cross context virtual CPU structure.
3120	* @param pVmcsInfo The VMCS info. object.
3121	*/
3122	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3123	{
3124	PVM pVM = pVCpu->CTX_SUFF(pVM);
3125	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3126	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3127
3128	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3129	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3130
3131	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3132	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3133
3134	/* Enable the VMX-preemption timer. */
3135	if (pVM->hm.s.vmx.fUsePreemptTimer)
3136	{
3137	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3138	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3139	}
3140
3141	#if 0
3142	/* Enable posted-interrupt processing. */
3143	if (pVM->hm.s.fPostedIntrs)
3144	{
3145	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3146	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3147	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3148	}
3149	#endif
3150
3151	if ((fVal & fZap) != fVal)
3152	{
3153	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3154	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3155	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3156	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3157	}
3158
3159	/* Commit it to the VMCS and update our cache. */
3160	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3161	AssertRCReturn(rc, rc);
3162	pVmcsInfo->u32PinCtls = fVal;
3163
3164	return VINF_SUCCESS;
3165	}
3166
3167
3168	/**
3169	* Sets up secondary processor-based VM-execution controls in the VMCS.
3170	*
3171	* @returns VBox status code.
3172	* @param pVCpu The cross context virtual CPU structure.
3173	* @param pVmcsInfo The VMCS info. object.
3174	*/
3175	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3176	{
3177	PVM pVM = pVCpu->CTX_SUFF(pVM);
3178	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3179	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3180
3181	/* WBINVD causes a VM-exit. */
3182	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3183	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3184
3185	/* Enable EPT (aka nested-paging). */
3186	if (pVM->hm.s.fNestedPaging)
3187	fVal \|= VMX_PROC_CTLS2_EPT;
3188
3189	/* Enable the INVPCID instruction if supported by the hardware and we expose
3190	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
3191	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
3192	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
3193	fVal \|= VMX_PROC_CTLS2_INVPCID;
3194
3195	/* Enable VPID. */
3196	if (pVM->hm.s.vmx.fVpid)
3197	fVal \|= VMX_PROC_CTLS2_VPID;
3198
3199	/* Enable unrestricted guest execution. */
3200	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3201	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3202
3203	#if 0
3204	if (pVM->hm.s.fVirtApicRegs)
3205	{
3206	/* Enable APIC-register virtualization. */
3207	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3208	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3209
3210	/* Enable virtual-interrupt delivery. */
3211	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3212	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3213	}
3214	#endif
3215
3216	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3217	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3218	* done dynamically. */
3219	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3220	{
3221	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3222	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3223	AssertRCReturn(rc, rc);
3224	}
3225
3226	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3227	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3228	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3229	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3230	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3231
3232	/* Enable Pause-Loop exiting. */
3233	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3234	&& pVM->hm.s.vmx.cPleGapTicks
3235	&& pVM->hm.s.vmx.cPleWindowTicks)
3236	{
3237	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3238
3239	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3240	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3241	AssertRCReturn(rc, rc);
3242	}
3243
3244	if ((fVal & fZap) != fVal)
3245	{
3246	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3247	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3248	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3249	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3250	}
3251
3252	/* Commit it to the VMCS and update our cache. */
3253	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3254	AssertRCReturn(rc, rc);
3255	pVmcsInfo->u32ProcCtls2 = fVal;
3256
3257	return VINF_SUCCESS;
3258	}
3259
3260
3261	/**
3262	* Sets up processor-based VM-execution controls in the VMCS.
3263	*
3264	* @returns VBox status code.
3265	* @param pVCpu The cross context virtual CPU structure.
3266	* @param pVmcsInfo The VMCS info. object.
3267	*/
3268	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3269	{
3270	PVM pVM = pVCpu->CTX_SUFF(pVM);
3271
3272	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3273	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3274
3275	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3276	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3277	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3278	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3279	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3280	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3281	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3282
3283	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3284	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3285	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3286	{
3287	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3288	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3289	}
3290
3291	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3292	if (!pVM->hm.s.fNestedPaging)
3293	{
3294	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3295	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3296	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3297	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3298	}
3299
3300	/* Use TPR shadowing if supported by the CPU. */
3301	if ( PDMHasApic(pVM)
3302	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3303	{
3304	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3305	/* CR8 writes cause a VM-exit based on TPR threshold. */
3306	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3307	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3308	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3309	AssertRCReturn(rc, rc);
3310	}
3311	else
3312	{
3313	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3314	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3315	if (pVM->hm.s.fAllow64BitGuests)
3316	{
3317	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3318	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3319	}
3320	}
3321
3322	/* Use MSR-bitmaps if supported by the CPU. */
3323	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3324	{
3325	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3326	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3327	AssertRCReturn(rc, rc);
3328	}
3329
3330	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3331	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3332	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3333
3334	if ((fVal & fZap) != fVal)
3335	{
3336	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3337	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3338	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3339	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3340	}
3341
3342	/* Commit it to the VMCS and update our cache. */
3343	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3344	AssertRCReturn(rc, rc);
3345	pVmcsInfo->u32ProcCtls = fVal;
3346
3347	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3348	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3349	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3350
3351	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3352	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3353	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3354
3355	/* Sanity check, should not really happen. */
3356	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3357	{ /* likely */ }
3358	else
3359	{
3360	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3361	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362	}
3363
3364	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3365	return VINF_SUCCESS;
3366	}
3367
3368
3369	/**
3370	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3371	* Processor-based VM-execution) control fields in the VMCS.
3372	*
3373	* @returns VBox status code.
3374	* @param pVCpu The cross context virtual CPU structure.
3375	* @param pVmcsInfo The VMCS info. object.
3376	*/
3377	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3378	{
3379	/* Set the auto-load/store MSR area addresses in the VMCS. */
3380	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3381	if (RT_SUCCESS(rc))
3382	{
3383	/* Set the VMCS link pointer in the VMCS. */
3384	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3385	if (RT_SUCCESS(rc))
3386	{
3387	/* Set the CR0/CR4 guest/host mask. */
3388	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3389	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3390	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3391	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3392	if (RT_SUCCESS(rc))
3393	{
3394	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3395	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3396	return VINF_SUCCESS;
3397	}
3398	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3399	}
3400	else
3401	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3402	}
3403	else
3404	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3405	return rc;
3406	}
3407
3408
3409	/**
3410	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3411	*
3412	* We shall setup those exception intercepts that don't change during the
3413	* lifetime of the VM here. The rest are done dynamically while loading the
3414	* guest state.
3415	*
3416	* @returns VBox status code.
3417	* @param pVCpu The cross context virtual CPU structure.
3418	* @param pVmcsInfo The VMCS info. object.
3419	*/
3420	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3421	{
3422	/*
3423	* The following exceptions are always intercepted:
3424	*
3425	* #AC - To prevent the guest from hanging the CPU.
3426	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3427	* recursive #DBs can cause a CPU hang.
3428	* #PF - To sync our shadow page tables when nested-paging is not used.
3429	*/
3430	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3431	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3432	\| RT_BIT(X86_XCPT_DB)
3433	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3434
3435	/* Commit it to the VMCS. */
3436	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3437	AssertRCReturn(rc, rc);
3438
3439	/* Update our cache of the exception bitmap. */
3440	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3441	return VINF_SUCCESS;
3442	}
3443
3444
3445	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3446	/**
3447	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3448	*
3449	* @returns VBox status code.
3450	* @param pVCpu The cross context virtual CPU structure.
3451	* @param pVmcsInfo The VMCS info. object.
3452	*/
3453	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3454	{
3455	PVM pVM = pVCpu->CTX_SUFF(pVM);
3456	int rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3457	if (RT_SUCCESS(rc))
3458	{
3459	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3460	if (RT_SUCCESS(rc))
3461	{
3462	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3463	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3464	if (RT_SUCCESS(rc))
3465	{
3466	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3467	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3468	if (RT_SUCCESS(rc))
3469	return VINF_SUCCESS;
3470
3471	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3472	}
3473	else
3474	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3475	}
3476	else
3477	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3478	}
3479	else
3480	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3481
3482	return rc;
3483	}
3484	#endif
3485
3486
3487	/**
3488	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3489	* VMX.
3490	*
3491	* @returns VBox status code.
3492	* @param pVCpu The cross context virtual CPU structure.
3493	* @param pVmcsInfo The VMCS info. object.
3494	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3495	*/
3496	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3497	{
3498	Assert(pVmcsInfo);
3499	Assert(pVmcsInfo->pvVmcs);
3500	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3501
3502	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3503	PVM pVM = pVCpu->CTX_SUFF(pVM);
3504	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3505	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3506
3507	LogFlowFunc(("\n"));
3508
3509	/*
3510	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3511	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3512	*/
3513	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3514	if (RT_SUCCESS(rc))
3515	{
3516	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3517	if (RT_SUCCESS(rc))
3518	{
3519	if (!fIsNstGstVmcs)
3520	{
3521	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3522	if (RT_SUCCESS(rc))
3523	{
3524	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3525	if (RT_SUCCESS(rc))
3526	{
3527	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3528	if (RT_SUCCESS(rc))
3529	{
3530	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3531	if (RT_SUCCESS(rc))
3532	{ /* likely */ }
3533	else
3534	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3535	}
3536	else
3537	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3538	}
3539	else
3540	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3541	}
3542	else
3543	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3544	}
3545	else
3546	{
3547	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3548	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3549	if (RT_SUCCESS(rc))
3550	{ /* likely */ }
3551	else
3552	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3553	#else
3554	AssertFailed();
3555	#endif
3556	}
3557	}
3558	else
3559	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3560	}
3561	else
3562	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3563
3564	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3565	if (RT_SUCCESS(rc))
3566	{
3567	rc = hmR0VmxClearVmcs(pVmcsInfo);
3568	if (RT_SUCCESS(rc))
3569	{ /* likely */ }
3570	else
3571	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3572	}
3573
3574	/*
3575	* Update the last-error record both for failures and success, so we
3576	* can propagate the status code back to ring-3 for diagnostics.
3577	*/
3578	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3579	NOREF(pszVmcs);
3580	return rc;
3581	}
3582
3583
3584	/**
3585	* Does global VT-x initialization (called during module initialization).
3586	*
3587	* @returns VBox status code.
3588	*/
3589	VMMR0DECL(int) VMXR0GlobalInit(void)
3590	{
3591	#ifdef HMVMX_USE_FUNCTION_TABLE
3592	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3593	# ifdef VBOX_STRICT
3594	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3595	Assert(g_apfnVMExitHandlers[i]);
3596	# endif
3597	#endif
3598	return VINF_SUCCESS;
3599	}
3600
3601
3602	/**
3603	* Does global VT-x termination (called during module termination).
3604	*/
3605	VMMR0DECL(void) VMXR0GlobalTerm()
3606	{
3607	/* Nothing to do currently. */
3608	}
3609
3610
3611	/**
3612	* Sets up and activates VT-x on the current CPU.
3613	*
3614	* @returns VBox status code.
3615	* @param pHostCpu The HM physical-CPU structure.
3616	* @param pVM The cross context VM structure. Can be
3617	* NULL after a host resume operation.
3618	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3619	* fEnabledByHost is @c true).
3620	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3621	* @a fEnabledByHost is @c true).
3622	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3623	* enable VT-x on the host.
3624	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3625	*/
3626	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3627	PCSUPHWVIRTMSRS pHwvirtMsrs)
3628	{
3629	Assert(pHostCpu);
3630	Assert(pHwvirtMsrs);
3631	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3632
3633	/* Enable VT-x if it's not already enabled by the host. */
3634	if (!fEnabledByHost)
3635	{
3636	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3637	if (RT_FAILURE(rc))
3638	return rc;
3639	}
3640
3641	/*
3642	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3643	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3644	* invalidated when flushing by VPID.
3645	*/
3646	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3647	{
3648	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3649	pHostCpu->fFlushAsidBeforeUse = false;
3650	}
3651	else
3652	pHostCpu->fFlushAsidBeforeUse = true;
3653
3654	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3655	++pHostCpu->cTlbFlushes;
3656
3657	return VINF_SUCCESS;
3658	}
3659
3660
3661	/**
3662	* Deactivates VT-x on the current CPU.
3663	*
3664	* @returns VBox status code.
3665	* @param pvCpuPage Pointer to the VMXON region.
3666	* @param HCPhysCpuPage Physical address of the VMXON region.
3667	*
3668	* @remarks This function should never be called when SUPR0EnableVTx() or
3669	* similar was used to enable VT-x on the host.
3670	*/
3671	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3672	{
3673	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3674
3675	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3676	return hmR0VmxLeaveRootMode();
3677	}
3678
3679
3680	/**
3681	* Does per-VM VT-x initialization.
3682	*
3683	* @returns VBox status code.
3684	* @param pVM The cross context VM structure.
3685	*/
3686	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3687	{
3688	LogFlowFunc(("pVM=%p\n", pVM));
3689
3690	int rc = hmR0VmxStructsAlloc(pVM);
3691	if (RT_FAILURE(rc))
3692	{
3693	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3694	return rc;
3695	}
3696
3697	return VINF_SUCCESS;
3698	}
3699
3700
3701	/**
3702	* Does per-VM VT-x termination.
3703	*
3704	* @returns VBox status code.
3705	* @param pVM The cross context VM structure.
3706	*/
3707	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3708	{
3709	LogFlowFunc(("pVM=%p\n", pVM));
3710
3711	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3712	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3713	{
3714	Assert(pVM->hm.s.vmx.pvScratch);
3715	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3716	}
3717	#endif
3718	hmR0VmxStructsFree(pVM);
3719	return VINF_SUCCESS;
3720	}
3721
3722
3723	/**
3724	* Sets up the VM for execution using hardware-assisted VMX.
3725	* This function is only called once per-VM during initialization.
3726	*
3727	* @returns VBox status code.
3728	* @param pVM The cross context VM structure.
3729	*/
3730	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3731	{
3732	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3733	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3734
3735	LogFlowFunc(("pVM=%p\n", pVM));
3736
3737	/*
3738	* At least verify if VMX is enabled, since we can't check if we're in
3739	* VMX root mode or not without causing a #GP.
3740	*/
3741	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3742	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3743	{ /* likely */ }
3744	else
3745	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3746
3747	/*
3748	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3749	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3750	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3751	*/
3752	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3753	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3754	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3755	{
3756	LogRelFunc(("Invalid real-on-v86 state.\n"));
3757	return VERR_INTERNAL_ERROR;
3758	}
3759
3760	/* Initialize these always, see hmR3InitFinalizeR0().*/
3761	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3762	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3763
3764	/* Setup the tagged-TLB flush handlers. */
3765	int rc = hmR0VmxSetupTaggedTlb(pVM);
3766	if (RT_FAILURE(rc))
3767	{
3768	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3769	return rc;
3770	}
3771
3772	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3773	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3774	#if HC_ARCH_BITS == 64
3775	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3776	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3777	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3778	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3779	#endif
3780
3781	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3782	{
3783	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3784	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3785
3786	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3787	if (RT_SUCCESS(rc))
3788	{
3789	#if HC_ARCH_BITS == 32
3790	hmR0VmxInitVmcsReadCache(pVCpu);
3791	#endif
3792	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3793	if (pVM->cpum.ro.GuestFeatures.fVmx)
3794	{
3795	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3796	if (RT_SUCCESS(rc))
3797	{ /* likely */ }
3798	else
3799	{
3800	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3801	return rc;
3802	}
3803	}
3804	#endif
3805	}
3806	else
3807	{
3808	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3809	return rc;
3810	}
3811	}
3812
3813	return VINF_SUCCESS;
3814	}
3815
3816
3817	#if HC_ARCH_BITS == 32
3818	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3819	/**
3820	* Check if guest state allows safe use of 32-bit switcher again.
3821	*
3822	* Segment bases and protected mode structures must be 32-bit addressable
3823	* because the 32-bit switcher will ignore high dword when writing these VMCS
3824	* fields. See @bugref{8432} for details.
3825	*
3826	* @returns true if safe, false if must continue to use the 64-bit switcher.
3827	* @param pCtx Pointer to the guest-CPU context.
3828	*
3829	* @remarks No-long-jump zone!!!
3830	*/
3831	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3832	{
3833	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3834	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3835	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3836	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3837	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3838	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3839	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3840	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3841	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3842	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3843
3844	/* All good, bases are 32-bit. */
3845	return true;
3846	}
3847	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3848
3849	# ifdef VBOX_STRICT
3850	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3851	{
3852	switch (idxField)
3853	{
3854	case VMX_VMCS_GUEST_RIP:
3855	case VMX_VMCS_GUEST_RSP:
3856	case VMX_VMCS_GUEST_SYSENTER_EIP:
3857	case VMX_VMCS_GUEST_SYSENTER_ESP:
3858	case VMX_VMCS_GUEST_GDTR_BASE:
3859	case VMX_VMCS_GUEST_IDTR_BASE:
3860	case VMX_VMCS_GUEST_CS_BASE:
3861	case VMX_VMCS_GUEST_DS_BASE:
3862	case VMX_VMCS_GUEST_ES_BASE:
3863	case VMX_VMCS_GUEST_FS_BASE:
3864	case VMX_VMCS_GUEST_GS_BASE:
3865	case VMX_VMCS_GUEST_SS_BASE:
3866	case VMX_VMCS_GUEST_LDTR_BASE:
3867	case VMX_VMCS_GUEST_TR_BASE:
3868	case VMX_VMCS_GUEST_CR3:
3869	return true;
3870	}
3871	return false;
3872	}
3873
3874	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3875	{
3876	switch (idxField)
3877	{
3878	/* Read-only fields. */
3879	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3880	return true;
3881	}
3882	/* Remaining readable fields should also be writable. */
3883	return hmR0VmxIsValidWriteField(idxField);
3884	}
3885	# endif /* VBOX_STRICT */
3886
3887
3888	/**
3889	* Executes the specified handler in 64-bit mode.
3890	*
3891	* @returns VBox status code (no informational status codes).
3892	* @param pVCpu The cross context virtual CPU structure.
3893	* @param enmOp The operation to perform.
3894	* @param cParams Number of parameters.
3895	* @param paParam Array of 32-bit parameters.
3896	*/
3897	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3898	{
3899	PVM pVM = pVCpu->CTX_SUFF(pVM);
3900	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3901	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3902	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3903	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3904
3905	#ifdef VBOX_STRICT
3906	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3907	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3908
3909	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3910	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3911	#endif
3912
3913	/* Disable interrupts. */
3914	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3915
3916	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3917	RTCPUID idHostCpu = RTMpCpuId();
3918	CPUMR0SetLApic(pVCpu, idHostCpu);
3919	#endif
3920
3921	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3922
3923	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3924	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3925
3926	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3927	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3928	hmR0VmxClearVmcs(pVmcsInfo);
3929
3930	/* Leave VMX root mode and disable VMX. */
3931	VMXDisable();
3932	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3933
3934	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3935	CPUMSetHyperEIP(pVCpu, enmOp);
3936	for (int i = (int)cParams - 1; i >= 0; i--)
3937	CPUMPushHyper(pVCpu, paParam[i]);
3938
3939	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3940
3941	/* Call the switcher. */
3942	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3943	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3944
3945	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3946	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3947
3948	/* Re-enter VMX root mode. */
3949	int rc2 = VMXEnable(HCPhysCpuPage);
3950	if (RT_FAILURE(rc2))
3951	{
3952	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3953	ASMSetFlags(fOldEFlags);
3954	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3955	return rc2;
3956	}
3957
3958	/* Restore the VMCS as the current VMCS. */
3959	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3960	AssertRC(rc2);
3961	Assert(!(ASMGetFlags() & X86_EFL_IF));
3962	ASMSetFlags(fOldEFlags);
3963	return rc;
3964	}
3965
3966
3967	/**
3968	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3969	* supporting 64-bit guests.
3970	*
3971	* @returns VBox status code.
3972	* @param fResume Whether to VMLAUNCH or VMRESUME.
3973	* @param pCtx Pointer to the guest-CPU context.
3974	* @param pCache Pointer to the VMCS batch cache.
3975	* @param pVM The cross context VM structure.
3976	* @param pVCpu The cross context virtual CPU structure.
3977	*/
3978	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3979	{
3980	NOREF(fResume);
3981
3982	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3983	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3984	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3985
3986	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3987	pCache->uPos = 1;
3988	pCache->interPD = PGMGetInterPaeCR3(pVM);
3989	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3990	#endif
3991
3992	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3993	pCache->TestIn.HCPhysCpuPage = 0;
3994	pCache->TestIn.HCPhysVmcs = 0;
3995	pCache->TestIn.pCache = 0;
3996	pCache->TestOut.HCPhysVmcs = 0;
3997	pCache->TestOut.pCache = 0;
3998	pCache->TestOut.pCtx = 0;
3999	pCache->TestOut.eflags = 0;
4000	#else
4001	NOREF(pCache);
4002	#endif
4003
4004	uint32_t aParam[10];
4005	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4006	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4007	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4008	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4009	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4010	aParam[5] = 0;
4011	aParam[6] = VM_RC_ADDR(pVM, pVM);
4012	aParam[7] = 0;
4013	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4014	aParam[9] = 0;
4015
4016	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4017	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4018	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4019	#endif
4020	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4021
4022	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4023	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4024	Assert(pCtx->dr[4] == 10);
4025	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4026	#endif
4027
4028	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4029	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4030	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4031	pVmcsInfo->HCPhysVmcs));
4032	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4033	pCache->TestOut.HCPhysVmcs));
4034	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4035	pCache->TestOut.pCache));
4036	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4037	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4038	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4039	pCache->TestOut.pCtx));
4040	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4041	#endif
4042	NOREF(pCtx);
4043	return rc;
4044	}
4045	#endif
4046
4047
4048	/**
4049	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4050	* the VMCS.
4051	*
4052	* @returns VBox status code.
4053	*/
4054	static int hmR0VmxExportHostControlRegs(void)
4055	{
4056	RTCCUINTREG uReg = ASMGetCR0();
4057	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4058	AssertRCReturn(rc, rc);
4059
4060	uReg = ASMGetCR3();
4061	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4062	AssertRCReturn(rc, rc);
4063
4064	uReg = ASMGetCR4();
4065	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4066	AssertRCReturn(rc, rc);
4067	return rc;
4068	}
4069
4070
4071	/**
4072	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4073	* the host-state area in the VMCS.
4074	*
4075	* @returns VBox status code.
4076	* @param pVCpu The cross context virtual CPU structure.
4077	*/
4078	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4079	{
4080	#if HC_ARCH_BITS == 64
4081	/**
4082	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4083	* requirements. See hmR0VmxExportHostSegmentRegs().
4084	*/
4085	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4086	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4087	{ \
4088	bool fValidSelector = true; \
4089	if ((selValue) & X86_SEL_LDT) \
4090	{ \
4091	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4092	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4093	} \
4094	if (fValidSelector) \
4095	{ \
4096	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4097	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4098	} \
4099	(selValue) = 0; \
4100	}
4101
4102	/*
4103	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4104	* will be messed up. We should -not- save the messed up state without restoring
4105	* the original host-state, see @bugref{7240}.
4106	*
4107	* This apparently can happen (most likely the FPU changes), deal with it rather than
4108	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4109	*/
4110	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4111	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4112	{
4113	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4114	pVCpu->idCpu));
4115	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4116	}
4117	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4118	#else
4119	RT_NOREF(pVCpu);
4120	#endif
4121
4122	/*
4123	* Host DS, ES, FS and GS segment registers.
4124	*/
4125	#if HC_ARCH_BITS == 64
4126	RTSEL uSelDS = ASMGetDS();
4127	RTSEL uSelES = ASMGetES();
4128	RTSEL uSelFS = ASMGetFS();
4129	RTSEL uSelGS = ASMGetGS();
4130	#else
4131	RTSEL uSelDS = 0;
4132	RTSEL uSelES = 0;
4133	RTSEL uSelFS = 0;
4134	RTSEL uSelGS = 0;
4135	#endif
4136
4137	/*
4138	* Host CS and SS segment registers.
4139	*/
4140	RTSEL uSelCS = ASMGetCS();
4141	RTSEL uSelSS = ASMGetSS();
4142
4143	/*
4144	* Host TR segment register.
4145	*/
4146	RTSEL uSelTR = ASMGetTR();
4147
4148	#if HC_ARCH_BITS == 64
4149	/*
4150	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4151	* gain VM-entry and restore them before we get preempted.
4152	*
4153	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4154	*/
4155	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4156	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4157	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4158	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4159	# undef VMXLOCAL_ADJUST_HOST_SEG
4160	#endif
4161
4162	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4163	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4164	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4165	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4166	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4167	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4168	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4169	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4170	Assert(uSelCS);
4171	Assert(uSelTR);
4172
4173	/* Write these host selector fields into the host-state area in the VMCS. */
4174	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4175	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4176	#if HC_ARCH_BITS == 64
4177	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4178	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4179	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4180	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4181	#else
4182	NOREF(uSelDS);
4183	NOREF(uSelES);
4184	NOREF(uSelFS);
4185	NOREF(uSelGS);
4186	#endif
4187	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4188	AssertRCReturn(rc, rc);
4189
4190	/*
4191	* Host GDTR and IDTR.
4192	*/
4193	RTGDTR Gdtr;
4194	RTIDTR Idtr;
4195	RT_ZERO(Gdtr);
4196	RT_ZERO(Idtr);
4197	ASMGetGDTR(&Gdtr);
4198	ASMGetIDTR(&Idtr);
4199	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4200	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4201	AssertRCReturn(rc, rc);
4202
4203	#if HC_ARCH_BITS == 64
4204	/*
4205	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4206	* them to the maximum limit (0xffff) on every VM-exit.
4207	*/
4208	if (Gdtr.cbGdt != 0xffff)
4209	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4210
4211	/*
4212	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4213	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4214	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4215	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4216	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4217	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4218	* at 0xffff on hosts where we are sure it won't cause trouble.
4219	*/
4220	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4221	if (Idtr.cbIdt < 0x0fff)
4222	# else
4223	if (Idtr.cbIdt != 0xffff)
4224	# endif
4225	{
4226	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4227	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4228	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4229	}
4230	#endif
4231
4232	/*
4233	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4234	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4235	* RPL should be too in most cases.
4236	*/
4237	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4238	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4239
4240	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4241	#if HC_ARCH_BITS == 64
4242	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4243
4244	/*
4245	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4246	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4247	* restoration if the host has something else. Task switching is not supported in 64-bit
4248	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4249	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4250	*
4251	* [1] See Intel spec. 3.5 "System Descriptor Types".
4252	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4253	*/
4254	PVM pVM = pVCpu->CTX_SUFF(pVM);
4255	Assert(pDesc->System.u4Type == 11);
4256	if ( pDesc->System.u16LimitLow != 0x67
4257	\|\| pDesc->System.u4LimitHigh)
4258	{
4259	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4260	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4261	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4262	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4263	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4264	}
4265
4266	/*
4267	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4268	*/
4269	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4270	{
4271	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4272	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4273	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4274	{
4275	/* The GDT is read-only but the writable GDT is available. */
4276	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4277	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4278	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4279	AssertRCReturn(rc, rc);
4280	}
4281	}
4282	#else
4283	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4284	#endif
4285	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4286	AssertRCReturn(rc, rc);
4287
4288	/*
4289	* Host FS base and GS base.
4290	*/
4291	#if HC_ARCH_BITS == 64
4292	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4293	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4294	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4295	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4296	AssertRCReturn(rc, rc);
4297
4298	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4299	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4300	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4301	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4302	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4303	#endif
4304	return VINF_SUCCESS;
4305	}
4306
4307
4308	/**
4309	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4310	* host-state area of the VMCS.
4311	*
4312	* These MSRs will be automatically restored on the host after every successful
4313	* VM-exit.
4314	*
4315	* @returns VBox status code.
4316	* @param pVCpu The cross context virtual CPU structure.
4317	*
4318	* @remarks No-long-jump zone!!!
4319	*/
4320	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4321	{
4322	AssertPtr(pVCpu);
4323
4324	/*
4325	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4326	* rather than swapping them on every VM-entry.
4327	*/
4328	hmR0VmxLazySaveHostMsrs(pVCpu);
4329
4330	/*
4331	* Host Sysenter MSRs.
4332	*/
4333	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4334	#if HC_ARCH_BITS == 32
4335	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4336	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4337	#else
4338	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4339	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4340	#endif
4341	AssertRCReturn(rc, rc);
4342
4343	/*
4344	* Host EFER MSR.
4345	*
4346	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4347	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4348	*/
4349	PVM pVM = pVCpu->CTX_SUFF(pVM);
4350	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4351	{
4352	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4353	AssertRCReturn(rc, rc);
4354	}
4355
4356	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4357	* hmR0VmxExportGuestEntryExitCtls(). */
4358
4359	return VINF_SUCCESS;
4360	}
4361
4362
4363	/**
4364	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4365	*
4366	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4367	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4368	*
4369	* @returns true if we need to load guest EFER, false otherwise.
4370	* @param pVCpu The cross context virtual CPU structure.
4371	*
4372	* @remarks Requires EFER, CR4.
4373	* @remarks No-long-jump zone!!!
4374	*/
4375	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4376	{
4377	#ifdef HMVMX_ALWAYS_SWAP_EFER
4378	RT_NOREF(pVCpu);
4379	return true;
4380	#else
4381	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4382	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4383	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4384	if (CPUMIsGuestInLongModeEx(pCtx))
4385	return false;
4386	#endif
4387
4388	PVM pVM = pVCpu->CTX_SUFF(pVM);
4389	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4390	uint64_t const u64GuestEfer = pCtx->msrEFER;
4391
4392	/*
4393	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4394	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4395	*/
4396	if ( CPUMIsGuestInLongModeEx(pCtx)
4397	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4398	return true;
4399
4400	/*
4401	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4402	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4403	*
4404	* See Intel spec. 4.5 "IA-32e Paging".
4405	* See Intel spec. 4.1.1 "Three Paging Modes".
4406	*
4407	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4408	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4409	*/
4410	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4411	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4412	if ( (pCtx->cr4 & X86_CR4_PAE)
4413	&& (pCtx->cr0 & X86_CR0_PG)
4414	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4415	{
4416	/* Assert that host is NX capable. */
4417	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4418	return true;
4419	}
4420
4421	return false;
4422	#endif
4423	}
4424
4425	/**
4426	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4427	* VMCS.
4428	*
4429	* This is typically required when the guest changes paging mode.
4430	*
4431	* @returns VBox status code.
4432	* @param pVCpu The cross context virtual CPU structure.
4433	* @param pVmxTransient The VMX-transient structure.
4434	*
4435	* @remarks Requires EFER.
4436	* @remarks No-long-jump zone!!!
4437	*/
4438	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4439	{
4440	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4441	{
4442	PVM pVM = pVCpu->CTX_SUFF(pVM);
4443	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4444
4445	/*
4446	* VM-entry controls.
4447	*/
4448	{
4449	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4450	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4451
4452	/*
4453	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4454	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4455	*
4456	* For nested-guests, this is a mandatory VM-entry control. It's also
4457	* required because we do not want to leak host bits to the nested-guest.
4458	*/
4459	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4460
4461	/*
4462	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4463	*
4464	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4465	* required to get the nested-guest working with hardware-assisted VMX execution.
4466	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4467	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4468	* here rather than while merging the guest VMCS controls.
4469	*/
4470	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4471	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4472	else
4473	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4474
4475	/*
4476	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4477	*
4478	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4479	* regardless of whether the nested-guest VMCS specifies it because we are free to
4480	* load whatever MSRs we require and we do not need to modify the guest visible copy
4481	* of the VM-entry MSR load area.
4482	*/
4483	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4484	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4485	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4486	else
4487	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4488
4489	/*
4490	* The following should -not- be set (since we're not in SMM mode):
4491	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4492	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4493	*/
4494
4495	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4496	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4497
4498	if ((fVal & fZap) == fVal)
4499	{ /* likely */ }
4500	else
4501	{
4502	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4503	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4504	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4505	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4506	}
4507
4508	/* Commit it to the VMCS. */
4509	if (pVmcsInfo->u32EntryCtls != fVal)
4510	{
4511	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4512	AssertRCReturn(rc, rc);
4513	pVmcsInfo->u32EntryCtls = fVal;
4514	}
4515	}
4516
4517	/*
4518	* VM-exit controls.
4519	*/
4520	{
4521	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4522	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4523
4524	/*
4525	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4526	* supported the 1-setting of this bit.
4527	*
4528	* For nested-guests, we set the "save debug controls" as the converse
4529	* "load debug controls" is mandatory for nested-guests anyway.
4530	*/
4531	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4532
4533	/*
4534	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4535	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4536	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4537	* hmR0VmxExportHostMsrs().
4538	*
4539	* For nested-guests, we always set this bit as we do not support 32-bit
4540	* hosts.
4541	*/
4542	#if HC_ARCH_BITS == 64
4543	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4544	#else
4545	Assert(!pVmxTransient->fIsNestedGuest);
4546	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4547	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4548	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4549	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4550	{
4551	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4552	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4553	}
4554	else
4555	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4556	#endif
4557
4558	/*
4559	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4560	*
4561	* For nested-guests, we should use the "save IA32_EFER" control if we also
4562	* used the "load IA32_EFER" control while exporting VM-entry controls.
4563	*/
4564	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4565	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4566	{
4567	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4568	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4569	}
4570
4571	/*
4572	* Enable saving of the VMX-preemption timer value on VM-exit.
4573	* For nested-guests, currently not exposed/used.
4574	*/
4575	if ( pVM->hm.s.vmx.fUsePreemptTimer
4576	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4577	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4578
4579	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4580	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4581
4582	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4583	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4584	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4585
4586	if ((fVal & fZap) == fVal)
4587	{ /* likely */ }
4588	else
4589	{
4590	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4591	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4592	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4593	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4594	}
4595
4596	/* Commit it to the VMCS. */
4597	if (pVmcsInfo->u32ExitCtls != fVal)
4598	{
4599	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4600	AssertRCReturn(rc, rc);
4601	pVmcsInfo->u32ExitCtls = fVal;
4602	}
4603	}
4604
4605	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4606	}
4607	return VINF_SUCCESS;
4608	}
4609
4610
4611	/**
4612	* Sets the TPR threshold in the VMCS.
4613	*
4614	* @returns VBox status code.
4615	* @param pVCpu The cross context virtual CPU structure.
4616	* @param pVmcsInfo The VMCS info. object.
4617	* @param u32TprThreshold The TPR threshold (task-priority class only).
4618	*/
4619	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4620	{
4621	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4622	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4623	RT_NOREF2(pVCpu, pVmcsInfo);
4624	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4625	}
4626
4627
4628	/**
4629	* Exports the guest APIC TPR state into the VMCS.
4630	*
4631	* @returns VBox status code.
4632	* @param pVCpu The cross context virtual CPU structure.
4633	* @param pVmxTransient The VMX-transient structure.
4634	*
4635	* @remarks No-long-jump zone!!!
4636	*/
4637	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4638	{
4639	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4640	{
4641	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4642
4643	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4644	if (!pVmxTransient->fIsNestedGuest)
4645	{
4646	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4647	&& APICIsEnabled(pVCpu))
4648	{
4649	/*
4650	* Setup TPR shadowing.
4651	*/
4652	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4653	{
4654	bool fPendingIntr = false;
4655	uint8_t u8Tpr = 0;
4656	uint8_t u8PendingIntr = 0;
4657	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4658	AssertRCReturn(rc, rc);
4659
4660	/*
4661	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4662	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4663	* priority of the pending interrupt so we can deliver the interrupt. If there
4664	* are no interrupts pending, set threshold to 0 to not cause any
4665	* TPR-below-threshold VM-exits.
4666	*/
4667	Assert(pVmcsInfo->pbVirtApic);
4668	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4669	uint32_t u32TprThreshold = 0;
4670	if (fPendingIntr)
4671	{
4672	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4673	(which is the Task-Priority Class). */
4674	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4675	const uint8_t u8TprPriority = u8Tpr >> 4;
4676	if (u8PendingPriority <= u8TprPriority)
4677	u32TprThreshold = u8PendingPriority;
4678	}
4679
4680	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4681	AssertRCReturn(rc, rc);
4682	}
4683	}
4684	}
4685	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4686	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4687	}
4688	return VINF_SUCCESS;
4689	}
4690
4691
4692	/**
4693	* Gets the guest interruptibility-state.
4694	*
4695	* @returns Guest's interruptibility-state.
4696	* @param pVCpu The cross context virtual CPU structure.
4697	* @param pVmxTransient The VMX-transient structure.
4698	*
4699	* @remarks No-long-jump zone!!!
4700	*/
4701	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4702	{
4703	/*
4704	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4705	*/
4706	uint32_t fIntrState = 0;
4707	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4708	{
4709	/* If inhibition is active, RIP and RFLAGS should've been updated
4710	(i.e. read previously from the VMCS or from ring-3). */
4711	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4712	#ifdef VBOX_STRICT
4713	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4714	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4715	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4716	#endif
4717	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4718	{
4719	if (pCtx->eflags.Bits.u1IF)
4720	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4721	else
4722	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4723	}
4724	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4725	{
4726	/*
4727	* We can clear the inhibit force flag as even if we go back to the recompiler
4728	* without executing guest code in VT-x, the flag's condition to be cleared is
4729	* met and thus the cleared state is correct.
4730	*/
4731	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4732	}
4733	}
4734
4735	/*
4736	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4737	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4738	* setting this would block host-NMIs and IRET will not clear the blocking.
4739	*
4740	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4741	*
4742	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4743	*/
4744	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
4745	&& CPUMIsGuestNmiBlocking(pVCpu))
4746	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4747
4748	return fIntrState;
4749	}
4750
4751
4752	/**
4753	* Exports the exception intercepts required for guest execution in the VMCS.
4754	*
4755	* @returns VBox status code.
4756	* @param pVCpu The cross context virtual CPU structure.
4757	* @param pVmxTransient The VMX-transient structure.
4758	*
4759	* @remarks No-long-jump zone!!!
4760	*/
4761	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4762	{
4763	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
4764	{
4765	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4766	if ( !pVmxTransient->fIsNestedGuest
4767	&& pVCpu->hm.s.fGIMTrapXcptUD)
4768	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4769	else
4770	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4771
4772	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4773	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
4774	}
4775	return VINF_SUCCESS;
4776	}
4777
4778
4779	/**
4780	* Exports the guest's RIP into the guest-state area in the VMCS.
4781	*
4782	* @returns VBox status code.
4783	* @param pVCpu The cross context virtual CPU structure.
4784	*
4785	* @remarks No-long-jump zone!!!
4786	*/
4787	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4788	{
4789	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4790	{
4791	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4792
4793	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4794	AssertRCReturn(rc, rc);
4795
4796	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4797	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4798	}
4799	return VINF_SUCCESS;
4800	}
4801
4802
4803	/**
4804	* Exports the guest's RSP into the guest-state area in the VMCS.
4805	*
4806	* @returns VBox status code.
4807	* @param pVCpu The cross context virtual CPU structure.
4808	*
4809	* @remarks No-long-jump zone!!!
4810	*/
4811	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4812	{
4813	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4814	{
4815	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4816
4817	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4818	AssertRCReturn(rc, rc);
4819
4820	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4821	}
4822	return VINF_SUCCESS;
4823	}
4824
4825
4826	/**
4827	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4828	*
4829	* @returns VBox status code.
4830	* @param pVCpu The cross context virtual CPU structure.
4831	* @param pVmxTransient The VMX-transient structure.
4832	*
4833	* @remarks No-long-jump zone!!!
4834	*/
4835	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4836	{
4837	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4838	{
4839	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4840
4841	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4842	Let us assert it as such and use 32-bit VMWRITE. */
4843	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4844	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4845	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4846	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4847
4848	/*
4849	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4850	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4851	* can run the real-mode guest code under Virtual 8086 mode.
4852	*/
4853	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4854	if (pVmcsInfo->RealMode.fRealOnV86Active)
4855	{
4856	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4857	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4858	Assert(!pVmxTransient->fIsNestedGuest);
4859	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4860	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4861	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4862	}
4863
4864	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4865	AssertRCReturn(rc, rc);
4866
4867	/*
4868	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4869	*
4870	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4871	* through the hypervisor debugger using EFLAGS.TF.
4872	*/
4873	if ( !pVmxTransient->fIsNestedGuest
4874	&& !pVCpu->hm.s.fSingleInstruction
4875	&& fEFlags.Bits.u1TF)
4876	{
4877	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4878	* premature trips to ring-3 esp since IEM does not yet handle it. */
4879	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4880	AssertRCReturn(rc, rc);
4881	}
4882	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4883	* nested-guest VMCS. */
4884
4885	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4886	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4887	}
4888	return VINF_SUCCESS;
4889	}
4890
4891
4892	/**
4893	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4894	*
4895	* The guest FPU state is always pre-loaded hence we don't need to bother about
4896	* sharing FPU related CR0 bits between the guest and host.
4897	*
4898	* @returns VBox status code.
4899	* @param pVCpu The cross context virtual CPU structure.
4900	* @param pVmxTransient The VMX-transient structure.
4901	*
4902	* @remarks No-long-jump zone!!!
4903	*/
4904	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4905	{
4906	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4907	{
4908	PVM pVM = pVCpu->CTX_SUFF(pVM);
4909	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4910
4911	/*
4912	* Figure out fixed CR0 bits in VMX operation.
4913	*/
4914	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4915	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4916	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4917	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4918	else
4919	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4920
4921	if (!pVmxTransient->fIsNestedGuest)
4922	{
4923	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4924	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4925	uint64_t const u64ShadowCr0 = u64GuestCr0;
4926	Assert(!RT_HI_U32(u64GuestCr0));
4927
4928	/*
4929	* Setup VT-x's view of the guest CR0.
4930	*/
4931	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4932	if (pVM->hm.s.fNestedPaging)
4933	{
4934	if (CPUMIsGuestPagingEnabled(pVCpu))
4935	{
4936	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4937	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4938	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4939	}
4940	else
4941	{
4942	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4943	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4944	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4945	}
4946
4947	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4948	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4949	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4950	}
4951	else
4952	{
4953	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4954	u64GuestCr0 \|= X86_CR0_WP;
4955	}
4956
4957	/*
4958	* Guest FPU bits.
4959	*
4960	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4961	* using CR0.TS.
4962	*
4963	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4964	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4965	*/
4966	u64GuestCr0 \|= X86_CR0_NE;
4967
4968	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4969	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4970
4971	/*
4972	* Update exception intercepts.
4973	*/
4974	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4975	if (pVmcsInfo->RealMode.fRealOnV86Active)
4976	{
4977	Assert(PDMVmmDevHeapIsEnabled(pVM));
4978	Assert(pVM->hm.s.vmx.pRealModeTSS);
4979	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4980	}
4981	else
4982	{
4983	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4984	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4985	if (fInterceptMF)
4986	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4987	}
4988
4989	/* Additional intercepts for debugging, define these yourself explicitly. */
4990	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4991	uXcptBitmap \|= 0
4992	\| RT_BIT(X86_XCPT_BP)
4993	\| RT_BIT(X86_XCPT_DE)
4994	\| RT_BIT(X86_XCPT_NM)
4995	\| RT_BIT(X86_XCPT_TS)
4996	\| RT_BIT(X86_XCPT_UD)
4997	\| RT_BIT(X86_XCPT_NP)
4998	\| RT_BIT(X86_XCPT_SS)
4999	\| RT_BIT(X86_XCPT_GP)
5000	\| RT_BIT(X86_XCPT_PF)
5001	\| RT_BIT(X86_XCPT_MF)
5002	;
5003	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5004	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5005	#endif
5006	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5007	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5008	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5009
5010	/* Apply the fixed CR0 bits and enable caching. */
5011	u64GuestCr0 \|= fSetCr0;
5012	u64GuestCr0 &= fZapCr0;
5013	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5014
5015	/* Commit the CR0 and related fields to the guest VMCS. */
5016	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5017	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5018	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5019	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5020	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5021	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5022	AssertRCReturn(rc, rc);
5023
5024	/* Update our caches. */
5025	pVmcsInfo->u32ProcCtls = uProcCtls;
5026	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5027
5028	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5029	}
5030	else
5031	{
5032	/*
5033	* With nested-guests, we may have extended the guest/host mask here since we
5034	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5035	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5036	* originally supplied. We must copy those bits from the nested-guest CR0 into
5037	* the nested-guest CR0 read-shadow.
5038	*/
5039	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5040	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5041	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5042	Assert(!RT_HI_U32(u64GuestCr0));
5043	Assert(u64GuestCr0 & X86_CR0_NE);
5044
5045	/* Apply the fixed CR0 bits and enable caching. */
5046	u64GuestCr0 \|= fSetCr0;
5047	u64GuestCr0 &= fZapCr0;
5048	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5049
5050	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5051	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5052	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5053	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5054	AssertRCReturn(rc, rc);
5055
5056	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5057	}
5058
5059	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5060	}
5061
5062	return VINF_SUCCESS;
5063	}
5064
5065
5066	/**
5067	* Exports the guest control registers (CR3, CR4) into the guest-state area
5068	* in the VMCS.
5069	*
5070	* @returns VBox strict status code.
5071	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5072	* without unrestricted guest access and the VMMDev is not presently
5073	* mapped (e.g. EFI32).
5074	*
5075	* @param pVCpu The cross context virtual CPU structure.
5076	* @param pVmxTransient The VMX-transient structure.
5077	*
5078	* @remarks No-long-jump zone!!!
5079	*/
5080	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5081	{
5082	int rc = VINF_SUCCESS;
5083	PVM pVM = pVCpu->CTX_SUFF(pVM);
5084
5085	/*
5086	* Guest CR2.
5087	* It's always loaded in the assembler code. Nothing to do here.
5088	*/
5089
5090	/*
5091	* Guest CR3.
5092	*/
5093	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5094	{
5095	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5096
5097	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
5098	if (pVM->hm.s.fNestedPaging)
5099	{
5100	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5101	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5102
5103	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5104	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5105	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5106	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5107
5108	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5109	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5110	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5111
5112	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5113	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5114	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5115	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5116	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5117	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5118	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5119
5120	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5121	AssertRCReturn(rc, rc);
5122
5123	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5124	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5125	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5126	{
5127	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5128	if (CPUMIsGuestInPAEModeEx(pCtx))
5129	{
5130	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5131	AssertRCReturn(rc, rc);
5132	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5133	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5134	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5135	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5136	AssertRCReturn(rc, rc);
5137	}
5138
5139	/*
5140	* The guest's view of its CR3 is unblemished with nested paging when the
5141	* guest is using paging or we have unrestricted guest execution to handle
5142	* the guest when it's not using paging.
5143	*/
5144	GCPhysGuestCR3 = pCtx->cr3;
5145	}
5146	else
5147	{
5148	/*
5149	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5150	* thinks it accesses physical memory directly, we use our identity-mapped
5151	* page table to map guest-linear to guest-physical addresses. EPT takes care
5152	* of translating it to host-physical addresses.
5153	*/
5154	RTGCPHYS GCPhys;
5155	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5156
5157	/* We obtain it here every time as the guest could have relocated this PCI region. */
5158	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5159	if (RT_SUCCESS(rc))
5160	{ /* likely */ }
5161	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5162	{
5163	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5164	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5165	}
5166	else
5167	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5168
5169	GCPhysGuestCR3 = GCPhys;
5170	}
5171
5172	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
5173	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
5174	AssertRCReturn(rc, rc);
5175	}
5176	else
5177	{
5178	/* Non-nested paging case, just use the hypervisor's CR3. */
5179	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
5180
5181	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
5182	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
5183	AssertRCReturn(rc, rc);
5184	}
5185
5186	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5187	}
5188
5189	/*
5190	* Guest CR4.
5191	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5192	*/
5193	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5194	{
5195	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5196	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5197
5198	/*
5199	* Figure out fixed CR4 bits in VMX operation.
5200	*/
5201	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5202	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5203
5204	/*
5205	* With nested-guests, we may have extended the guest/host mask here (since we
5206	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5207	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5208	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5209	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5210	*/
5211	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5212	uint64_t u64GuestCr4 = pCtx->cr4;
5213	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5214	? pCtx->cr4
5215	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5216	Assert(!RT_HI_U32(u64GuestCr4));
5217
5218	/*
5219	* Setup VT-x's view of the guest CR4.
5220	*
5221	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5222	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5223	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5224	*
5225	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5226	*/
5227	if (pVmcsInfo->RealMode.fRealOnV86Active)
5228	{
5229	Assert(pVM->hm.s.vmx.pRealModeTSS);
5230	Assert(PDMVmmDevHeapIsEnabled(pVM));
5231	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5232	}
5233
5234	if (pVM->hm.s.fNestedPaging)
5235	{
5236	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5237	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5238	{
5239	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5240	u64GuestCr4 \|= X86_CR4_PSE;
5241	/* Our identity mapping is a 32-bit page directory. */
5242	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5243	}
5244	/* else use guest CR4.*/
5245	}
5246	else
5247	{
5248	Assert(!pVmxTransient->fIsNestedGuest);
5249
5250	/*
5251	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5252	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5253	*/
5254	switch (pVCpu->hm.s.enmShadowMode)
5255	{
5256	case PGMMODE_REAL: /* Real-mode. */
5257	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5258	case PGMMODE_32_BIT: /* 32-bit paging. */
5259	{
5260	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5261	break;
5262	}
5263
5264	case PGMMODE_PAE: /* PAE paging. */
5265	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5266	{
5267	u64GuestCr4 \|= X86_CR4_PAE;
5268	break;
5269	}
5270
5271	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5272	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5273	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5274	break;
5275	#endif
5276	default:
5277	AssertFailed();
5278	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5279	}
5280	}
5281
5282	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5283	u64GuestCr4 \|= fSetCr4;
5284	u64GuestCr4 &= fZapCr4;
5285
5286	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5287	/** @todo Fix to 64-bit when we drop 32-bit. */
5288	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5289	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5290	AssertRCReturn(rc, rc);
5291
5292	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5293	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5294
5295	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5296
5297	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5298	}
5299	return rc;
5300	}
5301
5302
5303	/**
5304	* Exports the guest debug registers into the guest-state area in the VMCS.
5305	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5306	*
5307	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5308	*
5309	* @returns VBox status code.
5310	* @param pVCpu The cross context virtual CPU structure.
5311	* @param pVmxTransient The VMX-transient structure.
5312	*
5313	* @remarks No-long-jump zone!!!
5314	*/
5315	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5316	{
5317	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5318
5319	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5320	* stepping. */
5321	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5322	if (pVmxTransient->fIsNestedGuest)
5323	{
5324	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5325	AssertRCReturn(rc, rc);
5326	return VINF_SUCCESS;
5327	}
5328
5329	#ifdef VBOX_STRICT
5330	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5331	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5332	{
5333	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5334	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5335	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5336	}
5337	#endif
5338
5339	bool fSteppingDB = false;
5340	bool fInterceptMovDRx = false;
5341	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5342	if (pVCpu->hm.s.fSingleInstruction)
5343	{
5344	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5345	PVM pVM = pVCpu->CTX_SUFF(pVM);
5346	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5347	{
5348	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5349	Assert(fSteppingDB == false);
5350	}
5351	else
5352	{
5353	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5354	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5355	pVCpu->hm.s.fClearTrapFlag = true;
5356	fSteppingDB = true;
5357	}
5358	}
5359
5360	uint32_t u32GuestDr7;
5361	if ( fSteppingDB
5362	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5363	{
5364	/*
5365	* Use the combined guest and host DRx values found in the hypervisor register set
5366	* because the hypervisor debugger has breakpoints active or someone is single stepping
5367	* on the host side without a monitor trap flag.
5368	*
5369	* Note! DBGF expects a clean DR6 state before executing guest code.
5370	*/
5371	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5372	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5373	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5374	{
5375	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5376	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5377	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5378	}
5379	else
5380	#endif
5381	if (!CPUMIsHyperDebugStateActive(pVCpu))
5382	{
5383	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5384	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5385	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5386	}
5387
5388	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5389	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5390	pVCpu->hm.s.fUsingHyperDR7 = true;
5391	fInterceptMovDRx = true;
5392	}
5393	else
5394	{
5395	/*
5396	* If the guest has enabled debug registers, we need to load them prior to
5397	* executing guest code so they'll trigger at the right time.
5398	*/
5399	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5400	{
5401	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5402	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5403	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5404	{
5405	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5406	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5407	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5408	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5409	}
5410	else
5411	#endif
5412	if (!CPUMIsGuestDebugStateActive(pVCpu))
5413	{
5414	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5415	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5416	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5417	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5418	}
5419	Assert(!fInterceptMovDRx);
5420	}
5421	/*
5422	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5423	* must intercept #DB in order to maintain a correct DR6 guest value, and
5424	* because we need to intercept it to prevent nested #DBs from hanging the
5425	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5426	*/
5427	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5428	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5429	&& !CPUMIsGuestDebugStateActive(pVCpu))
5430	#else
5431	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5432	#endif
5433	{
5434	fInterceptMovDRx = true;
5435	}
5436
5437	/* Update DR7 with the actual guest value. */
5438	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5439	pVCpu->hm.s.fUsingHyperDR7 = false;
5440	}
5441
5442	if (fInterceptMovDRx)
5443	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5444	else
5445	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5446
5447	/*
5448	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5449	* monitor-trap flag and update our cache.
5450	*/
5451	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5452	{
5453	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5454	AssertRCReturn(rc2, rc2);
5455	pVmcsInfo->u32ProcCtls = uProcCtls;
5456	}
5457
5458	/*
5459	* Update guest DR7.
5460	*/
5461	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5462	AssertRCReturn(rc, rc);
5463
5464	/*
5465	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5466	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5467	*
5468	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5469	*/
5470	if (fSteppingDB)
5471	{
5472	Assert(pVCpu->hm.s.fSingleInstruction);
5473	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5474
5475	uint32_t fIntrState = 0;
5476	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5477	AssertRCReturn(rc, rc);
5478
5479	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5480	{
5481	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5482	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5483	AssertRCReturn(rc, rc);
5484	}
5485	}
5486
5487	return VINF_SUCCESS;
5488	}
5489
5490
5491	#ifdef VBOX_STRICT
5492	/**
5493	* Strict function to validate segment registers.
5494	*
5495	* @param pVCpu The cross context virtual CPU structure.
5496	* @param pVmcsInfo The VMCS info. object.
5497	*
5498	* @remarks Will import guest CR0 on strict builds during validation of
5499	* segments.
5500	*/
5501	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5502	{
5503	/*
5504	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5505	*
5506	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5507	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5508	* unusable bit and doesn't change the guest-context value.
5509	*/
5510	PVM pVM = pVCpu->CTX_SUFF(pVM);
5511	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5512	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5513	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5514	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5515	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5516	{
5517	/* Protected mode checks */
5518	/* CS */
5519	Assert(pCtx->cs.Attr.n.u1Present);
5520	Assert(!(pCtx->cs.Attr.u & 0xf00));
5521	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5522	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5523	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5524	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5525	\|\| (pCtx->cs.Attr.n.u1Granularity));
5526	/* CS cannot be loaded with NULL in protected mode. */
5527	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5528	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5529	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5530	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5531	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5532	else
5533	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5534	/* SS */
5535	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5536	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5537	if ( !(pCtx->cr0 & X86_CR0_PE)
5538	\|\| pCtx->cs.Attr.n.u4Type == 3)
5539	{
5540	Assert(!pCtx->ss.Attr.n.u2Dpl);
5541	}
5542	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5543	{
5544	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5545	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5546	Assert(pCtx->ss.Attr.n.u1Present);
5547	Assert(!(pCtx->ss.Attr.u & 0xf00));
5548	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5549	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5550	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5551	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5552	\|\| (pCtx->ss.Attr.n.u1Granularity));
5553	}
5554	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5555	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5556	{
5557	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5558	Assert(pCtx->ds.Attr.n.u1Present);
5559	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5560	Assert(!(pCtx->ds.Attr.u & 0xf00));
5561	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5562	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5563	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5564	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5565	\|\| (pCtx->ds.Attr.n.u1Granularity));
5566	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5567	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5568	}
5569	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5570	{
5571	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5572	Assert(pCtx->es.Attr.n.u1Present);
5573	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5574	Assert(!(pCtx->es.Attr.u & 0xf00));
5575	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5576	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5577	\|\| !(pCtx->es.Attr.n.u1Granularity));
5578	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5579	\|\| (pCtx->es.Attr.n.u1Granularity));
5580	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5581	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5582	}
5583	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5584	{
5585	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5586	Assert(pCtx->fs.Attr.n.u1Present);
5587	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5588	Assert(!(pCtx->fs.Attr.u & 0xf00));
5589	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5590	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5591	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5592	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5593	\|\| (pCtx->fs.Attr.n.u1Granularity));
5594	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5595	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5596	}
5597	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5598	{
5599	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5600	Assert(pCtx->gs.Attr.n.u1Present);
5601	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5602	Assert(!(pCtx->gs.Attr.u & 0xf00));
5603	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5604	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5605	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5606	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5607	\|\| (pCtx->gs.Attr.n.u1Granularity));
5608	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5609	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5610	}
5611	/* 64-bit capable CPUs. */
5612	# if HC_ARCH_BITS == 64
5613	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5614	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5615	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5616	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5617	# endif
5618	}
5619	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5620	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5621	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5622	{
5623	/* Real and v86 mode checks. */
5624	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5625	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5626	if (pVmcsInfo->RealMode.fRealOnV86Active)
5627	{
5628	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5629	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5630	}
5631	else
5632	{
5633	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5634	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5635	}
5636
5637	/* CS */
5638	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5639	Assert(pCtx->cs.u32Limit == 0xffff);
5640	Assert(u32CSAttr == 0xf3);
5641	/* SS */
5642	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5643	Assert(pCtx->ss.u32Limit == 0xffff);
5644	Assert(u32SSAttr == 0xf3);
5645	/* DS */
5646	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5647	Assert(pCtx->ds.u32Limit == 0xffff);
5648	Assert(u32DSAttr == 0xf3);
5649	/* ES */
5650	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5651	Assert(pCtx->es.u32Limit == 0xffff);
5652	Assert(u32ESAttr == 0xf3);
5653	/* FS */
5654	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5655	Assert(pCtx->fs.u32Limit == 0xffff);
5656	Assert(u32FSAttr == 0xf3);
5657	/* GS */
5658	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5659	Assert(pCtx->gs.u32Limit == 0xffff);
5660	Assert(u32GSAttr == 0xf3);
5661	/* 64-bit capable CPUs. */
5662	# if HC_ARCH_BITS == 64
5663	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5664	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5665	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5666	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5667	# endif
5668	}
5669	}
5670	#endif /* VBOX_STRICT */
5671
5672
5673	/**
5674	* Exports a guest segment register into the guest-state area in the VMCS.
5675	*
5676	* @returns VBox status code.
5677	* @param pVCpu The cross context virtual CPU structure.
5678	* @param pVmcsInfo The VMCS info. object.
5679	* @param iSegReg The segment register number (X86_SREG_XXX).
5680	* @param pSelReg Pointer to the segment selector.
5681	*
5682	* @remarks No-long-jump zone!!!
5683	*/
5684	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5685	{
5686	Assert(iSegReg < X86_SREG_COUNT);
5687	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5688	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5689	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5690	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5691
5692	uint32_t u32Access = pSelReg->Attr.u;
5693	if (pVmcsInfo->RealMode.fRealOnV86Active)
5694	{
5695	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5696	u32Access = 0xf3;
5697	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5698	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5699	RT_NOREF_PV(pVCpu);
5700	}
5701	else
5702	{
5703	/*
5704	* The way to differentiate between whether this is really a null selector or was just
5705	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5706	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5707	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5708	* NULL selectors loaded in protected-mode have their attribute as 0.
5709	*/
5710	if (!u32Access)
5711	u32Access = X86DESCATTR_UNUSABLE;
5712	}
5713
5714	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5715	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5716	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5717
5718	/*
5719	* Commit it to the VMCS.
5720	*/
5721	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5722	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5723	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5724	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5725	AssertRCReturn(rc, rc);
5726	return rc;
5727	}
5728
5729
5730	/**
5731	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5732	* area in the VMCS.
5733	*
5734	* @returns VBox status code.
5735	* @param pVCpu The cross context virtual CPU structure.
5736	* @param pVmxTransient The VMX-transient structure.
5737	*
5738	* @remarks Will import guest CR0 on strict builds during validation of
5739	* segments.
5740	* @remarks No-long-jump zone!!!
5741	*/
5742	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5743	{
5744	int rc = VERR_INTERNAL_ERROR_5;
5745	PVM pVM = pVCpu->CTX_SUFF(pVM);
5746	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5747	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5748
5749	/*
5750	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5751	*/
5752	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5753	{
5754	#ifdef VBOX_WITH_REM
5755	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5756	{
5757	Assert(!pVmxTransient->fIsNestedGuest);
5758	Assert(pVM->hm.s.vmx.pRealModeTSS);
5759	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5760	if ( pVmcsInfo->fWasInRealMode
5761	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5762	{
5763	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5764	in real-mode (e.g. OpenBSD 4.0) */
5765	REMFlushTBs(pVM);
5766	Log4Func(("Switch to protected mode detected!\n"));
5767	pVmcsInfo->fWasInRealMode = false;
5768	}
5769	}
5770	#endif
5771	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5772	{
5773	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5774	if (pVmcsInfo->RealMode.fRealOnV86Active)
5775	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5776	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5777	AssertRCReturn(rc, rc);
5778	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5779	}
5780
5781	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5782	{
5783	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5784	if (pVmcsInfo->RealMode.fRealOnV86Active)
5785	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5786	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5787	AssertRCReturn(rc, rc);
5788	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5789	}
5790
5791	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5792	{
5793	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5794	if (pVmcsInfo->RealMode.fRealOnV86Active)
5795	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5796	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5797	AssertRCReturn(rc, rc);
5798	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5799	}
5800
5801	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5802	{
5803	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5804	if (pVmcsInfo->RealMode.fRealOnV86Active)
5805	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5806	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5807	AssertRCReturn(rc, rc);
5808	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5809	}
5810
5811	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5812	{
5813	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5814	if (pVmcsInfo->RealMode.fRealOnV86Active)
5815	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5816	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5817	AssertRCReturn(rc, rc);
5818	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5819	}
5820
5821	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5822	{
5823	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5824	if (pVmcsInfo->RealMode.fRealOnV86Active)
5825	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5826	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5827	AssertRCReturn(rc, rc);
5828	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5829	}
5830
5831	#ifdef VBOX_STRICT
5832	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5833	#endif
5834	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5835	pCtx->cs.Attr.u));
5836	}
5837
5838	/*
5839	* Guest TR.
5840	*/
5841	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5842	{
5843	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5844
5845	/*
5846	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5847	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5848	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5849	*/
5850	uint16_t u16Sel;
5851	uint32_t u32Limit;
5852	uint64_t u64Base;
5853	uint32_t u32AccessRights;
5854	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5855	{
5856	u16Sel = pCtx->tr.Sel;
5857	u32Limit = pCtx->tr.u32Limit;
5858	u64Base = pCtx->tr.u64Base;
5859	u32AccessRights = pCtx->tr.Attr.u;
5860	}
5861	else
5862	{
5863	Assert(!pVmxTransient->fIsNestedGuest);
5864	Assert(pVM->hm.s.vmx.pRealModeTSS);
5865	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5866
5867	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5868	RTGCPHYS GCPhys;
5869	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5870	AssertRCReturn(rc, rc);
5871
5872	X86DESCATTR DescAttr;
5873	DescAttr.u = 0;
5874	DescAttr.n.u1Present = 1;
5875	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5876
5877	u16Sel = 0;
5878	u32Limit = HM_VTX_TSS_SIZE;
5879	u64Base = GCPhys;
5880	u32AccessRights = DescAttr.u;
5881	}
5882
5883	/* Validate. */
5884	Assert(!(u16Sel & RT_BIT(2)));
5885	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5886	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5887	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5888	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5889	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5890	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5891	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5892	Assert( (u32Limit & 0xfff) == 0xfff
5893	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5894	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5895	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5896
5897	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5898	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5899	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5900	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5901	AssertRCReturn(rc, rc);
5902
5903	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5904	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5905	}
5906
5907	/*
5908	* Guest GDTR.
5909	*/
5910	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5911	{
5912	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5913
5914	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5915	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5916	AssertRCReturn(rc, rc);
5917
5918	/* Validate. */
5919	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5920
5921	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5922	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5923	}
5924
5925	/*
5926	* Guest LDTR.
5927	*/
5928	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5929	{
5930	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5931
5932	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5933	uint32_t u32Access;
5934	if ( !pVmxTransient->fIsNestedGuest
5935	&& !pCtx->ldtr.Attr.u)
5936	u32Access = X86DESCATTR_UNUSABLE;
5937	else
5938	u32Access = pCtx->ldtr.Attr.u;
5939
5940	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5941	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5942	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5943	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5944	AssertRCReturn(rc, rc);
5945
5946	/* Validate. */
5947	if (!(u32Access & X86DESCATTR_UNUSABLE))
5948	{
5949	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5950	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5951	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5952	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5953	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5954	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5955	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5956	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5957	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5958	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5959	}
5960
5961	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5962	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5963	}
5964
5965	/*
5966	* Guest IDTR.
5967	*/
5968	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5969	{
5970	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5971
5972	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5973	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5974	AssertRCReturn(rc, rc);
5975
5976	/* Validate. */
5977	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5978
5979	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5980	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5981	}
5982
5983	return VINF_SUCCESS;
5984	}
5985
5986
5987	/**
5988	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5989	* areas.
5990	*
5991	* These MSRs will automatically be loaded to the host CPU on every successful
5992	* VM-entry and stored from the host CPU on every successful VM-exit.
5993	*
5994	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5995	* actual host MSR values are not- updated here for performance reasons. See
5996	* hmR0VmxExportHostMsrs().
5997	*
5998	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5999	*
6000	* @returns VBox status code.
6001	* @param pVCpu The cross context virtual CPU structure.
6002	* @param pVmxTransient The VMX-transient structure.
6003	*
6004	* @remarks No-long-jump zone!!!
6005	*/
6006	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6007	{
6008	AssertPtr(pVCpu);
6009	AssertPtr(pVmxTransient);
6010
6011	PVM pVM = pVCpu->CTX_SUFF(pVM);
6012	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6013
6014	/*
6015	* MSRs that we use the auto-load/store MSR area in the VMCS.
6016	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6017	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6018	*
6019	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6020	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6021	* emulation, nothing to do here.
6022	*/
6023	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6024	{
6025	if ( !pVmxTransient->fIsNestedGuest
6026	&& pVM->hm.s.fAllow64BitGuests)
6027	{
6028	#if HC_ARCH_BITS == 32
6029	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6030	Assert(!pVmxTransient->fIsNestedGuest);
6031
6032	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6033	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6034	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6035	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6036	AssertRCReturn(rc, rc);
6037	#endif
6038	}
6039	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6040	}
6041
6042	/*
6043	* Guest Sysenter MSRs.
6044	*/
6045	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6046	{
6047	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6048
6049	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6050	{
6051	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6052	AssertRCReturn(rc, rc);
6053	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6054	}
6055
6056	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6057	{
6058	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6059	AssertRCReturn(rc, rc);
6060	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6061	}
6062
6063	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6064	{
6065	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6066	AssertRCReturn(rc, rc);
6067	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6068	}
6069	}
6070
6071	/*
6072	* Guest/host EFER MSR.
6073	*/
6074	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6075	{
6076	/* Whether we are using the VMCS to swap the EFER MSR must have been
6077	determined earlier while exporting VM-entry/VM-exit controls. */
6078	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6079	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6080
6081	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6082	{
6083	/*
6084	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6085	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6086	*/
6087	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6088	{
6089	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6090	AssertRCReturn(rc, rc);
6091	}
6092	else
6093	{
6094	/*
6095	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6096	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6097	*/
6098	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6099	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6100	AssertRCReturn(rc, rc);
6101	}
6102	}
6103	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6104	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6105
6106	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6107	}
6108
6109	/*
6110	* Other MSRs.
6111	* Speculation Control (R/W).
6112	*/
6113	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6114	{
6115	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6116	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6117	{
6118	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6119	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6120	AssertRCReturn(rc, rc);
6121	}
6122	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6123	}
6124
6125	return VINF_SUCCESS;
6126	}
6127
6128
6129	/**
6130	* Selects up the appropriate function to run guest code.
6131	*
6132	* @returns VBox status code.
6133	* @param pVCpu The cross context virtual CPU structure.
6134	* @param pVmxTransient The VMX-transient structure.
6135	*
6136	* @remarks No-long-jump zone!!!
6137	*/
6138	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6139	{
6140	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6141	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6142
6143	if (CPUMIsGuestInLongModeEx(pCtx))
6144	{
6145	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6146	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6147	#endif
6148	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6149	#if HC_ARCH_BITS == 32
6150	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6151	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6152	{
6153	#ifdef VBOX_STRICT
6154	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6155	{
6156	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6157	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6158	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6159	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6160	("fCtxChanged=%#RX64\n", fCtxChanged));
6161	}
6162	#endif
6163	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6164
6165	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6166	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6167	pVmcsInfo->fSwitchedTo64on32 = true;
6168	Log4Func(("Selected 64-bit switcher\n"));
6169	}
6170	#else
6171	/* 64-bit host. */
6172	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6173	#endif
6174	}
6175	else
6176	{
6177	/* Guest is not in long mode, use the 32-bit handler. */
6178	#if HC_ARCH_BITS == 32
6179	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6180	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6181	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6182	{
6183	# ifdef VBOX_STRICT
6184	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6185	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6186	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6187	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6188	("fCtxChanged=%#RX64\n", fCtxChanged));
6189	# endif
6190	}
6191	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6192	/*
6193	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6194	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6195	* switcher flag now because we know the guest is in a sane state where it's safe
6196	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6197	* the much faster 32-bit switcher again.
6198	*/
6199	if (!pVmcsInfo->fSwitchedTo64on32)
6200	{
6201	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6202	Log4Func(("Selected 32-bit switcher\n"));
6203	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6204	}
6205	else
6206	{
6207	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6208	if ( pVmcsInfo->RealMode.fRealOnV86Active
6209	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6210	{
6211	pVmcsInfo->fSwitchedTo64on32 = false;
6212	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6213	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6214	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6215	\| HM_CHANGED_HOST_CONTEXT);
6216	Log4Func(("Selected 32-bit switcher (safe)\n"));
6217	}
6218	}
6219	# else
6220	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6221	# endif
6222	#else
6223	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6224	#endif
6225	}
6226	Assert(pVmcsInfo->pfnStartVM);
6227	return VINF_SUCCESS;
6228	}
6229
6230
6231	/**
6232	* Wrapper for running the guest code in VT-x.
6233	*
6234	* @returns VBox status code, no informational status codes.
6235	* @param pVCpu The cross context virtual CPU structure.
6236	* @param pVmxTransient The VMX-transient structure.
6237	*
6238	* @remarks No-long-jump zone!!!
6239	*/
6240	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6241	{
6242	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6243	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6244	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6245
6246	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6247
6248	/*
6249	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6250	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6251	* callee-saved and thus the need for this XMM wrapper.
6252	*
6253	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6254	*/
6255	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6256	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6257	PVM pVM = pVCpu->CTX_SUFF(pVM);
6258	#ifdef VBOX_WITH_KERNEL_USING_XMM
6259	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6260	#else
6261	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6262	#endif
6263	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6264	return rc;
6265	}
6266
6267
6268	/**
6269	* Reports world-switch error and dumps some useful debug info.
6270	*
6271	* @param pVCpu The cross context virtual CPU structure.
6272	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6273	* @param pVmxTransient The VMX-transient structure (only
6274	* exitReason updated).
6275	*/
6276	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6277	{
6278	Assert(pVCpu);
6279	Assert(pVmxTransient);
6280	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6281
6282	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6283	switch (rcVMRun)
6284	{
6285	case VERR_VMX_INVALID_VMXON_PTR:
6286	AssertFailed();
6287	break;
6288	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6289	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6290	{
6291	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6292	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6293	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6294	AssertRC(rc);
6295
6296	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6297	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6298	Cannot do it here as we may have been long preempted. */
6299
6300	#ifdef VBOX_STRICT
6301	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6302	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6303	pVmxTransient->uExitReason));
6304	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6305	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6306	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6307	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6308	else
6309	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6310	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6311	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6312
6313	/* VMX control bits. */
6314	uint32_t u32Val;
6315	uint64_t u64Val;
6316	RTHCUINTREG uHCReg;
6317	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6318	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6319	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6320	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6321	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6322	{
6323	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6324	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6325	}
6326	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6327	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6328	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6329	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6330	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6331	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6332	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6333	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6334	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6335	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6336	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6337	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6338	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6339	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6340	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6341	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6342	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6343	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6344	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6345	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6346	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6347	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6348	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6349	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6350	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6351	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6352	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6353	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6354	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6355	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6356	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6357	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6358	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6359	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6360	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6361	{
6362	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6363	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6364	}
6365
6366	/* Guest bits. */
6367	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6368	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6369	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6370	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6371	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6372	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6373	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6374	{
6375	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6376	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6377	}
6378
6379	/* Host bits. */
6380	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6381	Log4(("Host CR0 %#RHr\n", uHCReg));
6382	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6383	Log4(("Host CR3 %#RHr\n", uHCReg));
6384	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6385	Log4(("Host CR4 %#RHr\n", uHCReg));
6386
6387	RTGDTR HostGdtr;
6388	PCX86DESCHC pDesc;
6389	ASMGetGDTR(&HostGdtr);
6390	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6391	Log4(("Host CS %#08x\n", u32Val));
6392	if (u32Val < HostGdtr.cbGdt)
6393	{
6394	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6395	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6396	}
6397
6398	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6399	Log4(("Host DS %#08x\n", u32Val));
6400	if (u32Val < HostGdtr.cbGdt)
6401	{
6402	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6403	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6404	}
6405
6406	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6407	Log4(("Host ES %#08x\n", u32Val));
6408	if (u32Val < HostGdtr.cbGdt)
6409	{
6410	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6411	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6412	}
6413
6414	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6415	Log4(("Host FS %#08x\n", u32Val));
6416	if (u32Val < HostGdtr.cbGdt)
6417	{
6418	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6419	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6420	}
6421
6422	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6423	Log4(("Host GS %#08x\n", u32Val));
6424	if (u32Val < HostGdtr.cbGdt)
6425	{
6426	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6427	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6428	}
6429
6430	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6431	Log4(("Host SS %#08x\n", u32Val));
6432	if (u32Val < HostGdtr.cbGdt)
6433	{
6434	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6435	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6436	}
6437
6438	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6439	Log4(("Host TR %#08x\n", u32Val));
6440	if (u32Val < HostGdtr.cbGdt)
6441	{
6442	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6443	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6444	}
6445
6446	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6447	Log4(("Host TR Base %#RHv\n", uHCReg));
6448	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6449	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6450	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6451	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6452	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6453	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6454	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6455	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6456	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6457	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6458	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6459	Log4(("Host RSP %#RHv\n", uHCReg));
6460	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6461	Log4(("Host RIP %#RHv\n", uHCReg));
6462	# if HC_ARCH_BITS == 64
6463	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6464	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6465	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6466	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6467	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6468	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6469	# endif
6470	#endif /* VBOX_STRICT */
6471	break;
6472	}
6473
6474	default:
6475	/* Impossible */
6476	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6477	break;
6478	}
6479	}
6480
6481
6482	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6483	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6484	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6485	# endif
6486
6487	/**
6488	* Initialize the VMCS-Read cache.
6489	*
6490	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6491	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6492	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6493	* (those that have a 32-bit FULL & HIGH part).
6494	*
6495	* @param pVCpu The cross context virtual CPU structure.
6496	*/
6497	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6498	{
6499	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6500	do { \
6501	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6502	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6503	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6504	++cReadFields; \
6505	} while (0)
6506
6507	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6508	uint32_t cReadFields = 0;
6509
6510	/*
6511	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6512	* and serve to indicate exceptions to the rules.
6513	*/
6514
6515	/* Guest-natural selector base fields. */
6516	#if 0
6517	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6518	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6520	#endif
6521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6530	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6531	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6532	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6533	#if 0
6534	/* Unused natural width guest-state fields. */
6535	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6537	#endif
6538	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6539	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6540
6541	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6542	these 64-bit fields (using "FULL" and "HIGH" fields). */
6543	#if 0
6544	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6545	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6546	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6547	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6548	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6549	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6550	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6551	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6552	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6553	#endif
6554
6555	/* Natural width guest-state fields. */
6556	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6557	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6558
6559	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6560	{
6561	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6562	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6563	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6564	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6565	}
6566	else
6567	{
6568	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6569	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6570	}
6571
6572	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6573	}
6574
6575
6576	/**
6577	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6578	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6579	* darwin, running 64-bit guests).
6580	*
6581	* @returns VBox status code.
6582	* @param pVCpu The cross context virtual CPU structure.
6583	* @param idxField The VMCS field encoding.
6584	* @param u64Val 16, 32 or 64-bit value.
6585	*/
6586	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6587	{
6588	int rc;
6589	switch (idxField)
6590	{
6591	/*
6592	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6593	*/
6594	/* 64-bit Control fields. */
6595	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6596	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6597	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6598	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6599	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6600	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6601	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6602	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6603	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6604	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6605	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6606	case VMX_VMCS64_CTRL_EPTP_FULL:
6607	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6608	/* 64-bit Guest-state fields. */
6609	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6610	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6611	case VMX_VMCS64_GUEST_PAT_FULL:
6612	case VMX_VMCS64_GUEST_EFER_FULL:
6613	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6614	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6615	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6616	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6617	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6618	/* 64-bit Host-state fields. */
6619	case VMX_VMCS64_HOST_PAT_FULL:
6620	case VMX_VMCS64_HOST_EFER_FULL:
6621	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6622	{
6623	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6624	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6625	break;
6626	}
6627
6628	/*
6629	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6630	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6631	*/
6632	/* Natural-width Guest-state fields. */
6633	case VMX_VMCS_GUEST_CR3:
6634	case VMX_VMCS_GUEST_ES_BASE:
6635	case VMX_VMCS_GUEST_CS_BASE:
6636	case VMX_VMCS_GUEST_SS_BASE:
6637	case VMX_VMCS_GUEST_DS_BASE:
6638	case VMX_VMCS_GUEST_FS_BASE:
6639	case VMX_VMCS_GUEST_GS_BASE:
6640	case VMX_VMCS_GUEST_LDTR_BASE:
6641	case VMX_VMCS_GUEST_TR_BASE:
6642	case VMX_VMCS_GUEST_GDTR_BASE:
6643	case VMX_VMCS_GUEST_IDTR_BASE:
6644	case VMX_VMCS_GUEST_RSP:
6645	case VMX_VMCS_GUEST_RIP:
6646	case VMX_VMCS_GUEST_SYSENTER_ESP:
6647	case VMX_VMCS_GUEST_SYSENTER_EIP:
6648	{
6649	if (!(RT_HI_U32(u64Val)))
6650	{
6651	/* If this field is 64-bit, VT-x will zero out the top bits. */
6652	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6653	}
6654	else
6655	{
6656	/* Assert that only the 32->64 switcher case should ever come here. */
6657	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6658	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6659	}
6660	break;
6661	}
6662
6663	default:
6664	{
6665	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6666	pVCpu->hm.s.u32HMError = idxField;
6667	rc = VERR_INVALID_PARAMETER;
6668	break;
6669	}
6670	}
6671	AssertRCReturn(rc, rc);
6672	return rc;
6673	}
6674
6675
6676	/**
6677	* Queue up a VMWRITE by using the VMCS write cache.
6678	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6679	*
6680	* @param pVCpu The cross context virtual CPU structure.
6681	* @param idxField The VMCS field encoding.
6682	* @param u64Val 16, 32 or 64-bit value.
6683	*/
6684	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6685	{
6686	AssertPtr(pVCpu);
6687	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6688
6689	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6690	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6691
6692	/* Make sure there are no duplicates. */
6693	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6694	{
6695	if (pCache->Write.aField[i] == idxField)
6696	{
6697	pCache->Write.aFieldVal[i] = u64Val;
6698	return VINF_SUCCESS;
6699	}
6700	}
6701
6702	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6703	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6704	pCache->Write.cValidEntries++;
6705	return VINF_SUCCESS;
6706	}
6707	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6708
6709
6710	/**
6711	* Sets up the usage of TSC-offsetting and updates the VMCS.
6712	*
6713	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6714	* VMX-preemption timer.
6715	*
6716	* @returns VBox status code.
6717	* @param pVCpu The cross context virtual CPU structure.
6718	* @param pVmxTransient The VMX-transient structure.
6719	*
6720	* @remarks No-long-jump zone!!!
6721	*/
6722	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6723	{
6724	bool fOffsettedTsc;
6725	bool fParavirtTsc;
6726	uint64_t uTscOffset;
6727	PVM pVM = pVCpu->CTX_SUFF(pVM);
6728	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6729
6730	if (pVM->hm.s.vmx.fUsePreemptTimer)
6731	{
6732	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6733
6734	/* Make sure the returned values have sane upper and lower boundaries. */
6735	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6736	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6737	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6738	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6739
6740	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6741	* preemption timers here. We probably need to clamp the preemption timer,
6742	* after converting the timer value to the host. */
6743	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6744	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6745	AssertRC(rc);
6746	}
6747	else
6748	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6749
6750	if (fParavirtTsc)
6751	{
6752	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6753	information before every VM-entry, hence disable it for performance sake. */
6754	#if 0
6755	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6756	AssertRC(rc);
6757	#endif
6758	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6759	}
6760
6761	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6762	if ( fOffsettedTsc
6763	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6764	{
6765	if (pVmxTransient->fIsNestedGuest)
6766	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6767	if (pVmcsInfo->u64TscOffset != uTscOffset)
6768	{
6769	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6770	AssertRC(rc);
6771	pVmcsInfo->u64TscOffset = uTscOffset;
6772	}
6773
6774	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6775	{
6776	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6777	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6778	AssertRC(rc);
6779	pVmcsInfo->u32ProcCtls = uProcCtls;
6780	}
6781	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6782	}
6783	else
6784	{
6785	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6786	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6787	{
6788	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6789	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6790	AssertRC(rc);
6791	pVmcsInfo->u32ProcCtls = uProcCtls;
6792	}
6793	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6794	}
6795	}
6796
6797
6798	/**
6799	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6800	* VM-exit interruption info type.
6801	*
6802	* @returns The IEM exception flags.
6803	* @param uVector The event vector.
6804	* @param uVmxEventType The VMX event type.
6805	*
6806	* @remarks This function currently only constructs flags required for
6807	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6808	* and CR2 aspects of an exception are not included).
6809	*/
6810	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6811	{
6812	uint32_t fIemXcptFlags;
6813	switch (uVmxEventType)
6814	{
6815	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6816	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6817	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6818	break;
6819
6820	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6821	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6822	break;
6823
6824	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6825	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6826	break;
6827
6828	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6829	{
6830	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6831	if (uVector == X86_XCPT_BP)
6832	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6833	else if (uVector == X86_XCPT_OF)
6834	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6835	else
6836	{
6837	fIemXcptFlags = 0;
6838	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6839	}
6840	break;
6841	}
6842
6843	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6844	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6845	break;
6846
6847	default:
6848	fIemXcptFlags = 0;
6849	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6850	break;
6851	}
6852	return fIemXcptFlags;
6853	}
6854
6855
6856	/**
6857	* Sets an event as a pending event to be injected into the guest.
6858	*
6859	* @param pVCpu The cross context virtual CPU structure.
6860	* @param u32IntInfo The VM-entry interruption-information field.
6861	* @param cbInstr The VM-entry instruction length in bytes (for software
6862	* interrupts, exceptions and privileged software
6863	* exceptions).
6864	* @param u32ErrCode The VM-entry exception error code.
6865	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6866	* page-fault.
6867	*/
6868	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6869	RTGCUINTPTR GCPtrFaultAddress)
6870	{
6871	Assert(!pVCpu->hm.s.Event.fPending);
6872	pVCpu->hm.s.Event.fPending = true;
6873	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6874	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6875	pVCpu->hm.s.Event.cbInstr = cbInstr;
6876	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6877	}
6878
6879
6880	/**
6881	* Sets an external interrupt as pending-for-injection into the VM.
6882	*
6883	* @param pVCpu The cross context virtual CPU structure.
6884	* @param u8Interrupt The external interrupt vector.
6885	*/
6886	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6887	{
6888	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6889	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6890	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6891	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6892	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6893	}
6894
6895
6896	/**
6897	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6898	*
6899	* @param pVCpu The cross context virtual CPU structure.
6900	*/
6901	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6902	{
6903	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6904	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6905	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6906	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6907	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6908	}
6909
6910
6911	/**
6912	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6913	*
6914	* @param pVCpu The cross context virtual CPU structure.
6915	*/
6916	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6917	{
6918	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6919	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6920	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6921	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6922	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6923	}
6924
6925
6926	/**
6927	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6928	*
6929	* @param pVCpu The cross context virtual CPU structure.
6930	*/
6931	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6932	{
6933	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6934	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6935	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6936	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6937	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6938	}
6939
6940
6941	/**
6942	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6943	*
6944	* @param pVCpu The cross context virtual CPU structure.
6945	*/
6946	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6947	{
6948	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6949	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6950	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6951	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6952	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6953	}
6954
6955
6956	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6957	/**
6958	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6959	*
6960	* @param pVCpu The cross context virtual CPU structure.
6961	* @param u32ErrCode The error code for the general-protection exception.
6962	*/
6963	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6964	{
6965	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6966	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6967	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6968	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6969	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6970	}
6971
6972
6973	/**
6974	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6975	*
6976	* @param pVCpu The cross context virtual CPU structure.
6977	* @param u32ErrCode The error code for the stack exception.
6978	*/
6979	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6980	{
6981	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6982	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6983	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6984	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6985	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6986	}
6987
6988
6989	/**
6990	* Decodes the memory operand of an instruction that caused a VM-exit.
6991	*
6992	* The Exit qualification field provides the displacement field for memory
6993	* operand instructions, if any.
6994	*
6995	* @returns Strict VBox status code (i.e. informational status codes too).
6996	* @retval VINF_SUCCESS if the operand was successfully decoded.
6997	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6998	* operand.
6999	* @param pVCpu The cross context virtual CPU structure.
7000	* @param uExitInstrInfo The VM-exit instruction information field.
7001	* @param enmMemAccess The memory operand's access type (read or write).
7002	* @param GCPtrDisp The instruction displacement field, if any. For
7003	* RIP-relative addressing pass RIP + displacement here.
7004	* @param pGCPtrMem Where to store the effective destination memory address.
7005	*
7006	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
7007	* virtual-8086 mode hence skips those checks while verifying if the
7008	* segment is valid.
7009	*/
7010	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
7011	PRTGCPTR pGCPtrMem)
7012	{
7013	Assert(pGCPtrMem);
7014	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
7015	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
7016	\| CPUMCTX_EXTRN_CR0);
7017
7018	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
7019	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
7020	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
7021
7022	VMXEXITINSTRINFO ExitInstrInfo;
7023	ExitInstrInfo.u = uExitInstrInfo;
7024	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
7025	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
7026	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
7027	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
7028	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
7029	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
7030	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
7031	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
7032	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
7033
7034	/*
7035	* Validate instruction information.
7036	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
7037	*/
7038	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
7039	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
7040	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
7041	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
7042	AssertLogRelMsgReturn(fIsMemOperand,
7043	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
7044
7045	/*
7046	* Compute the complete effective address.
7047	*
7048	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
7049	* See AMD spec. 4.5.2 "Segment Registers".
7050	*/
7051	RTGCPTR GCPtrMem = GCPtrDisp;
7052	if (fBaseRegValid)
7053	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
7054	if (fIdxRegValid)
7055	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
7056
7057	RTGCPTR const GCPtrOff = GCPtrMem;
7058	if ( !fIsLongMode
7059	\|\| iSegReg >= X86_SREG_FS)
7060	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
7061	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
7062
7063	/*
7064	* Validate effective address.
7065	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
7066	*/
7067	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
7068	Assert(cbAccess > 0);
7069	if (fIsLongMode)
7070	{
7071	if (X86_IS_CANONICAL(GCPtrMem))
7072	{
7073	*pGCPtrMem = GCPtrMem;
7074	return VINF_SUCCESS;
7075	}
7076
7077	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
7078	* "Data Limit Checks in 64-bit Mode". */
7079	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
7080	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7081	return VINF_HM_PENDING_XCPT;
7082	}
7083
7084	/*
7085	* This is a watered down version of iemMemApplySegment().
7086	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
7087	* and segment CPL/DPL checks are skipped.
7088	*/
7089	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
7090	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
7091	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7092
7093	/* Check if the segment is present and usable. */
7094	if ( pSel->Attr.n.u1Present
7095	&& !pSel->Attr.n.u1Unusable)
7096	{
7097	Assert(pSel->Attr.n.u1DescType);
7098	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
7099	{
7100	/* Check permissions for the data segment. */
7101	if ( enmMemAccess == VMXMEMACCESS_WRITE
7102	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
7103	{
7104	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7105	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
7106	return VINF_HM_PENDING_XCPT;
7107	}
7108
7109	/* Check limits if it's a normal data segment. */
7110	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
7111	{
7112	if ( GCPtrFirst32 > pSel->u32Limit
7113	\|\| GCPtrLast32 > pSel->u32Limit)
7114	{
7115	Log4Func(("Data segment limit exceeded. "
7116	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7117	GCPtrLast32, pSel->u32Limit));
7118	if (iSegReg == X86_SREG_SS)
7119	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7120	else
7121	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7122	return VINF_HM_PENDING_XCPT;
7123	}
7124	}
7125	else
7126	{
7127	/* Check limits if it's an expand-down data segment.
7128	Note! The upper boundary is defined by the B bit, not the G bit! */
7129	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
7130	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
7131	{
7132	Log4Func(("Expand-down data segment limit exceeded. "
7133	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
7134	GCPtrLast32, pSel->u32Limit));
7135	if (iSegReg == X86_SREG_SS)
7136	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7137	else
7138	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7139	return VINF_HM_PENDING_XCPT;
7140	}
7141	}
7142	}
7143	else
7144	{
7145	/* Check permissions for the code segment. */
7146	if ( enmMemAccess == VMXMEMACCESS_WRITE
7147	\|\| ( enmMemAccess == VMXMEMACCESS_READ
7148	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
7149	{
7150	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
7151	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7152	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7153	return VINF_HM_PENDING_XCPT;
7154	}
7155
7156	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
7157	if ( GCPtrFirst32 > pSel->u32Limit
7158	\|\| GCPtrLast32 > pSel->u32Limit)
7159	{
7160	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
7161	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
7162	if (iSegReg == X86_SREG_SS)
7163	hmR0VmxSetPendingXcptSS(pVCpu, 0);
7164	else
7165	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7166	return VINF_HM_PENDING_XCPT;
7167	}
7168	}
7169	}
7170	else
7171	{
7172	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
7173	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7174	return VINF_HM_PENDING_XCPT;
7175	}
7176
7177	*pGCPtrMem = GCPtrMem;
7178	return VINF_SUCCESS;
7179	}
7180
7181
7182	/**
7183	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
7184	* guest attempting to execute a VMX instruction.
7185	*
7186	* @returns Strict VBox status code (i.e. informational status codes too).
7187	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
7188	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
7189	*
7190	* @param pVCpu The cross context virtual CPU structure.
7191	* @param uExitReason The VM-exit reason.
7192	*
7193	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
7194	* @remarks No-long-jump zone!!!
7195	*/
7196	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
7197	{
7198	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
7199	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7200
7201	/*
7202	* The physical CPU would have already checked the CPU mode/code segment.
7203	* We shall just assert here for paranoia.
7204	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7205	*/
7206	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
7207	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
7208	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
7209
7210	if (uExitReason == VMX_EXIT_VMXON)
7211	{
7212	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7213
7214	/*
7215	* We check CR4.VMXE because it is required to be always set while in VMX operation
7216	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
7217	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
7218	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
7219	*/
7220	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
7221	{
7222	Log4Func(("CR4.VMXE is not set -> #UD\n"));
7223	hmR0VmxSetPendingXcptUD(pVCpu);
7224	return VINF_HM_PENDING_XCPT;
7225	}
7226	}
7227	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
7228	{
7229	/*
7230	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7231	* (other than VMXON), we need to raise a #UD.
7232	*/
7233	Log4Func(("Not in VMX root mode -> #UD\n"));
7234	hmR0VmxSetPendingXcptUD(pVCpu);
7235	return VINF_HM_PENDING_XCPT;
7236	}
7237
7238	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
7239	return VINF_SUCCESS;
7240	}
7241	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7242
7243
7244	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7245	{
7246	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7247
7248	/*
7249	* If VT-x marks the segment as unusable, most other bits remain undefined:
7250	* - For CS the L, D and G bits have meaning.
7251	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7252	* - For the remaining data segments no bits are defined.
7253	*
7254	* The present bit and the unusable bit has been observed to be set at the
7255	* same time (the selector was supposed to be invalid as we started executing
7256	* a V8086 interrupt in ring-0).
7257	*
7258	* What should be important for the rest of the VBox code, is that the P bit is
7259	* cleared. Some of the other VBox code recognizes the unusable bit, but
7260	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7261	* safe side here, we'll strip off P and other bits we don't care about. If
7262	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7263	*
7264	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7265	*/
7266	#ifdef VBOX_STRICT
7267	uint32_t const uAttr = pSelReg->Attr.u;
7268	#endif
7269
7270	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7271	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7272	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7273
7274	#ifdef VBOX_STRICT
7275	VMMRZCallRing3Disable(pVCpu);
7276	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7277	# ifdef DEBUG_bird
7278	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7279	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7280	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7281	# endif
7282	VMMRZCallRing3Enable(pVCpu);
7283	NOREF(uAttr);
7284	#endif
7285	RT_NOREF2(pVCpu, idxSel);
7286	}
7287
7288
7289	/**
7290	* Imports a guest segment register from the current VMCS into the guest-CPU
7291	* context.
7292	*
7293	* @returns VBox status code.
7294	* @param pVCpu The cross context virtual CPU structure.
7295	* @param iSegReg The segment register number (X86_SREG_XXX).
7296	*
7297	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7298	* do not log!
7299	*/
7300	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7301	{
7302	Assert(iSegReg < X86_SREG_COUNT);
7303
7304	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7305	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7306	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7307	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7308	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7309	#else
7310	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7311	#endif
7312	uint64_t u64Base;
7313	uint32_t u32Sel, u32Limit, u32Attr;
7314	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7315	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7316	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7317	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7318	if (RT_SUCCESS(rc))
7319	{
7320	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7321	pSelReg->Sel = u32Sel;
7322	pSelReg->ValidSel = u32Sel;
7323	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7324	pSelReg->u32Limit = u32Limit;
7325	pSelReg->u64Base = u64Base;
7326	pSelReg->Attr.u = u32Attr;
7327	if (u32Attr & X86DESCATTR_UNUSABLE)
7328	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7329	}
7330	return rc;
7331	}
7332
7333
7334	/**
7335	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7336	*
7337	* @returns VBox status code.
7338	* @param pVCpu The cross context virtual CPU structure.
7339	*
7340	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7341	* do not log!
7342	*/
7343	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7344	{
7345	uint64_t u64Base;
7346	uint32_t u32Sel, u32Limit, u32Attr;
7347	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7348	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7349	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7350	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7351	if (RT_SUCCESS(rc))
7352	{
7353	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7354	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7355	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7356	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7357	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7358	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7359	if (u32Attr & X86DESCATTR_UNUSABLE)
7360	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7361	}
7362	return rc;
7363	}
7364
7365
7366	/**
7367	* Imports the guest TR from the current VMCS into the guest-CPU context.
7368	*
7369	* @returns VBox status code.
7370	* @param pVCpu The cross context virtual CPU structure.
7371	*
7372	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7373	* do not log!
7374	*/
7375	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7376	{
7377	uint32_t u32Sel, u32Limit, u32Attr;
7378	uint64_t u64Base;
7379	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7380	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7381	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7382	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7383	AssertRCReturn(rc, rc);
7384
7385	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7386	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7387	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7388	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7389	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7390	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7391	/* TR is the only selector that can never be unusable. */
7392	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7393	return VINF_SUCCESS;
7394	}
7395
7396
7397	/**
7398	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7399	*
7400	* @returns VBox status code.
7401	* @param pVCpu The cross context virtual CPU structure.
7402	*
7403	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7404	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7405	* instead!!!
7406	*/
7407	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7408	{
7409	uint64_t u64Val;
7410	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7411	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7412	{
7413	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7414	if (RT_SUCCESS(rc))
7415	{
7416	pCtx->rip = u64Val;
7417	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7418	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7419	}
7420	return rc;
7421	}
7422	return VINF_SUCCESS;
7423	}
7424
7425
7426	/**
7427	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7428	*
7429	* @returns VBox status code.
7430	* @param pVCpu The cross context virtual CPU structure.
7431	* @param pVmcsInfo The VMCS info. object.
7432	*
7433	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7434	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7435	* instead!!!
7436	*/
7437	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7438	{
7439	uint32_t u32Val;
7440	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7441	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7442	{
7443	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7444	if (RT_SUCCESS(rc))
7445	{
7446	pCtx->eflags.u32 = u32Val;
7447
7448	/* Restore eflags for real-on-v86-mode hack. */
7449	if (pVmcsInfo->RealMode.fRealOnV86Active)
7450	{
7451	pCtx->eflags.Bits.u1VM = 0;
7452	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7453	}
7454	}
7455	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7456	return rc;
7457	}
7458	return VINF_SUCCESS;
7459	}
7460
7461
7462	/**
7463	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7464	* context.
7465	*
7466	* @returns VBox status code.
7467	* @param pVCpu The cross context virtual CPU structure.
7468	* @param pVmcsInfo The VMCS info. object.
7469	*
7470	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7471	* do not log!
7472	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7473	* instead!!!
7474	*/
7475	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7476	{
7477	uint32_t u32Val;
7478	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7479	if (RT_SUCCESS(rc))
7480	{
7481	if (!u32Val)
7482	{
7483	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7484	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7485
7486	CPUMSetGuestNmiBlocking(pVCpu, false);
7487	}
7488	else
7489	{
7490	/*
7491	* We must import RIP here to set our EM interrupt-inhibited state.
7492	* We also import RFLAGS as our code that evaluates pending interrupts
7493	* before VM-entry requires it.
7494	*/
7495	rc = hmR0VmxImportGuestRip(pVCpu);
7496	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7497	if (RT_SUCCESS(rc))
7498	{
7499	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7500	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7501	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7502	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7503
7504	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7505	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7506	}
7507	}
7508	}
7509	return rc;
7510	}
7511
7512
7513	/**
7514	* Worker for VMXR0ImportStateOnDemand.
7515	*
7516	* @returns VBox status code.
7517	* @param pVCpu The cross context virtual CPU structure.
7518	* @param pVmcsInfo The VMCS info. object.
7519	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7520	*/
7521	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7522	{
7523	#define VMXLOCAL_BREAK_RC(a_rc) \
7524	if (RT_SUCCESS(a_rc)) \
7525	{ } \
7526	else \
7527	break
7528
7529	int rc = VINF_SUCCESS;
7530	PVM pVM = pVCpu->CTX_SUFF(pVM);
7531	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7532	uint64_t u64Val;
7533	uint32_t u32Val;
7534
7535	/*
7536	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7537	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7538	* neither are other host platforms.
7539	*
7540	* Committing this temporarily as it prevents BSOD.
7541	*
7542	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7543	*/
7544	#ifdef RT_OS_WINDOWS
7545	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7546	return VERR_HM_IPE_1;
7547	#endif
7548
7549	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7550
7551	/*
7552	* We disable interrupts to make the updating of the state and in particular
7553	* the fExtrn modification atomic wrt to preemption hooks.
7554	*/
7555	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7556
7557	fWhat &= pCtx->fExtrn;
7558	if (fWhat)
7559	{
7560	do
7561	{
7562	if (fWhat & CPUMCTX_EXTRN_RIP)
7563	{
7564	rc = hmR0VmxImportGuestRip(pVCpu);
7565	VMXLOCAL_BREAK_RC(rc);
7566	}
7567
7568	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7569	{
7570	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7571	VMXLOCAL_BREAK_RC(rc);
7572	}
7573
7574	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7575	{
7576	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7577	VMXLOCAL_BREAK_RC(rc);
7578	}
7579
7580	if (fWhat & CPUMCTX_EXTRN_RSP)
7581	{
7582	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7583	VMXLOCAL_BREAK_RC(rc);
7584	pCtx->rsp = u64Val;
7585	}
7586
7587	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7588	{
7589	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7590	if (fWhat & CPUMCTX_EXTRN_CS)
7591	{
7592	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7593	rc \|= hmR0VmxImportGuestRip(pVCpu);
7594	if (fRealOnV86Active)
7595	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7596	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7597	}
7598	if (fWhat & CPUMCTX_EXTRN_SS)
7599	{
7600	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7601	if (fRealOnV86Active)
7602	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7603	}
7604	if (fWhat & CPUMCTX_EXTRN_DS)
7605	{
7606	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7607	if (fRealOnV86Active)
7608	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7609	}
7610	if (fWhat & CPUMCTX_EXTRN_ES)
7611	{
7612	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7613	if (fRealOnV86Active)
7614	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7615	}
7616	if (fWhat & CPUMCTX_EXTRN_FS)
7617	{
7618	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7619	if (fRealOnV86Active)
7620	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7621	}
7622	if (fWhat & CPUMCTX_EXTRN_GS)
7623	{
7624	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7625	if (fRealOnV86Active)
7626	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7627	}
7628	VMXLOCAL_BREAK_RC(rc);
7629	}
7630
7631	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7632	{
7633	if (fWhat & CPUMCTX_EXTRN_LDTR)
7634	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7635
7636	if (fWhat & CPUMCTX_EXTRN_GDTR)
7637	{
7638	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7639	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7640	pCtx->gdtr.pGdt = u64Val;
7641	pCtx->gdtr.cbGdt = u32Val;
7642	}
7643
7644	/* Guest IDTR. */
7645	if (fWhat & CPUMCTX_EXTRN_IDTR)
7646	{
7647	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7648	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7649	pCtx->idtr.pIdt = u64Val;
7650	pCtx->idtr.cbIdt = u32Val;
7651	}
7652
7653	/* Guest TR. */
7654	if (fWhat & CPUMCTX_EXTRN_TR)
7655	{
7656	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7657	don't need to import that one. */
7658	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7659	rc \|= hmR0VmxImportGuestTr(pVCpu);
7660	}
7661	VMXLOCAL_BREAK_RC(rc);
7662	}
7663
7664	if (fWhat & CPUMCTX_EXTRN_DR7)
7665	{
7666	if (!pVCpu->hm.s.fUsingHyperDR7)
7667	{
7668	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7669	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7670	VMXLOCAL_BREAK_RC(rc);
7671	pCtx->dr[7] = u32Val;
7672	}
7673	}
7674
7675	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7676	{
7677	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7678	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7679	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7680	pCtx->SysEnter.cs = u32Val;
7681	VMXLOCAL_BREAK_RC(rc);
7682	}
7683
7684	#if HC_ARCH_BITS == 64
7685	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7686	{
7687	if ( pVM->hm.s.fAllow64BitGuests
7688	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7689	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7690	}
7691
7692	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7693	{
7694	if ( pVM->hm.s.fAllow64BitGuests
7695	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7696	{
7697	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7698	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7699	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7700	}
7701	}
7702	#endif
7703
7704	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7705	#if HC_ARCH_BITS == 32
7706	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7707	#endif
7708	)
7709	{
7710	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7711	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7712	Assert(pMsrs);
7713	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7714	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7715	for (uint32_t i = 0; i < cMsrs; i++)
7716	{
7717	uint32_t const idMsr = pMsrs[i].u32Msr;
7718	switch (idMsr)
7719	{
7720	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7721	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7722	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7723	#if HC_ARCH_BITS == 32
7724	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7725	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7726	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7727	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7728	#endif
7729	default:
7730	{
7731	pCtx->fExtrn = 0;
7732	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7733	ASMSetFlags(fEFlags);
7734	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7735	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7736	}
7737	}
7738	}
7739	}
7740
7741	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7742	{
7743	uint64_t u64Shadow;
7744	if (fWhat & CPUMCTX_EXTRN_CR0)
7745	{
7746	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7747	* remove when we drop 32-bit host w/ 64-bit host support, see
7748	* @bugref{9180#c39}. */
7749	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7750	#if HC_ARCH_BITS == 32
7751	uint32_t u32Shadow;
7752	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7753	u64Shadow = u32Shadow;
7754	#else
7755	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7756	#endif
7757	VMXLOCAL_BREAK_RC(rc);
7758	u64Val = u32Val;
7759	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7760	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7761
7762	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7763	CPUMSetGuestCR0(pVCpu, u64Val);
7764	VMMRZCallRing3Enable(pVCpu);
7765	}
7766
7767	if (fWhat & CPUMCTX_EXTRN_CR4)
7768	{
7769	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7770	* remove when we drop 32-bit host w/ 64-bit host support, see
7771	* @bugref{9180#c39}. */
7772	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7773	#if HC_ARCH_BITS == 32
7774	uint32_t u32Shadow;
7775	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7776	u64Shadow = u32Shadow;
7777	#else
7778	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7779	#endif
7780	VMXLOCAL_BREAK_RC(rc);
7781	u64Val = u32Val;
7782	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7783	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7784	pCtx->cr4 = u64Val;
7785	}
7786
7787	if (fWhat & CPUMCTX_EXTRN_CR3)
7788	{
7789	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7790	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7791	\|\| ( pVM->hm.s.fNestedPaging
7792	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7793	{
7794	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7795	VMXLOCAL_BREAK_RC(rc);
7796	if (pCtx->cr3 != u64Val)
7797	{
7798	pCtx->cr3 = u64Val;
7799	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7800	}
7801
7802	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7803	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7804	if (CPUMIsGuestInPAEModeEx(pCtx))
7805	{
7806	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7807	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7808	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7809	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7810	VMXLOCAL_BREAK_RC(rc);
7811	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7812	}
7813	}
7814	}
7815
7816	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7817	# if 0
7818	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
7819	* VM-exit handlers as parameters. We would handle it differently when using
7820	* the fast path. */
7821	/*
7822	* The hardware virtualization state currently consists of VMCS fields that may be
7823	* modified by execution of the nested-guest (that are not part of the general
7824	* guest state) and is visible to guest software. Hence, it is technically part of
7825	* the guest-CPU state when executing a nested-guest.
7826	*/
7827	if ( (fWhat & CPUMCTX_EXTRN_HWVIRT)
7828	&& CPUMIsGuestInVmxNonRootMode(pCtx))
7829	{
7830	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
7831	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
7832	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
7833	VMXLOCAL_BREAK_RC(rc);
7834
7835	/*
7836	* VM-entry can fail due to invalid-guest state, machine-check events and
7837	* MSR loading failures. Other than VM-exit reason and Exit qualification
7838	* all other VMCS fields are left unmodified on VM-entry failure.
7839	*
7840	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
7841	*/
7842	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
7843	if (!fEntryFailed)
7844	{
7845	/*
7846	* Some notes on VMCS fields that may need importing when the fast path
7847	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
7848	*
7849	* Requires fixing up when using hardware-assisted VMX:
7850	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
7851	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
7852	* - IDT-vectoring info: Think about this.
7853	* - IDT-vectoring error code: Think about this.
7854	*
7855	* Emulated:
7856	* - Guest-interruptiblity state: Derived from FFs and RIP.
7857	* - Guest pending debug exceptions: Derived from DR6.
7858	* - Guest activity state: Emulated from EM state.
7859	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
7860	* - Entry-interrupt info: Emulated, cleared to 0.
7861	*/
7862	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
7863	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
7864	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
7865	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
7866	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
7867	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
7868	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
7869	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
7870	/** @todo NSTVMX: Save and adjust preemption timer value. */
7871	}
7872
7873	VMXLOCAL_BREAK_RC(rc);
7874	}
7875	# endif
7876	#endif
7877	}
7878	} while (0);
7879
7880	if (RT_SUCCESS(rc))
7881	{
7882	/* Update fExtrn. */
7883	pCtx->fExtrn &= ~fWhat;
7884
7885	/* If everything has been imported, clear the HM keeper bit. */
7886	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7887	{
7888	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7889	Assert(!pCtx->fExtrn);
7890	}
7891	}
7892	}
7893	else
7894	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7895
7896	ASMSetFlags(fEFlags);
7897
7898	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7899
7900	if (RT_SUCCESS(rc))
7901	{ /* likely */ }
7902	else
7903	return rc;
7904
7905	/*
7906	* Honor any pending CR3 updates.
7907	*
7908	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7909	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7910	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7911	*
7912	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7913	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7914	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7915	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7916	*
7917	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7918	*/
7919	if (VMMRZCallRing3IsEnabled(pVCpu))
7920	{
7921	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7922	{
7923	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7924	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7925	}
7926
7927	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7928	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7929
7930	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7931	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7932	}
7933
7934	return VINF_SUCCESS;
7935	#undef VMXLOCAL_BREAK_RC
7936	}
7937
7938
7939	/**
7940	* Saves the guest state from the VMCS into the guest-CPU context.
7941	*
7942	* @returns VBox status code.
7943	* @param pVCpu The cross context virtual CPU structure.
7944	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7945	*/
7946	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7947	{
7948	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7949	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7950	}
7951
7952
7953	/**
7954	* Check per-VM and per-VCPU force flag actions that require us to go back to
7955	* ring-3 for one reason or another.
7956	*
7957	* @returns Strict VBox status code (i.e. informational status codes too)
7958	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7959	* ring-3.
7960	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7961	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7962	* interrupts)
7963	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7964	* all EMTs to be in ring-3.
7965	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7966	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7967	* to the EM loop.
7968	*
7969	* @param pVCpu The cross context virtual CPU structure.
7970	* @param fStepping Whether we are single-stepping the guest using the
7971	* hypervisor debugger.
7972	*/
7973	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7974	{
7975	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7976
7977	/*
7978	* Update pending interrupts into the APIC's IRR.
7979	*/
7980	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7981	APICUpdatePendingInterrupts(pVCpu);
7982
7983	/*
7984	* Anything pending? Should be more likely than not if we're doing a good job.
7985	*/
7986	PVM pVM = pVCpu->CTX_SUFF(pVM);
7987	if ( !fStepping
7988	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7989	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7990	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7991	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7992	return VINF_SUCCESS;
7993
7994	/* Pending PGM C3 sync. */
7995	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7996	{
7997	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7998	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7999	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8000	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8001	if (rcStrict2 != VINF_SUCCESS)
8002	{
8003	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8004	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8005	return rcStrict2;
8006	}
8007	}
8008
8009	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8010	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8011	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8012	{
8013	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8014	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8015	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8016	return rc2;
8017	}
8018
8019	/* Pending VM request packets, such as hardware interrupts. */
8020	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8021	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8022	{
8023	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8024	return VINF_EM_PENDING_REQUEST;
8025	}
8026
8027	/* Pending PGM pool flushes. */
8028	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8029	{
8030	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8031	return VINF_PGM_POOL_FLUSH_PENDING;
8032	}
8033
8034	/* Pending DMA requests. */
8035	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8036	{
8037	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8038	return VINF_EM_RAW_TO_R3;
8039	}
8040
8041	return VINF_SUCCESS;
8042	}
8043
8044
8045	/**
8046	* Converts any TRPM trap into a pending HM event. This is typically used when
8047	* entering from ring-3 (not longjmp returns).
8048	*
8049	* @param pVCpu The cross context virtual CPU structure.
8050	*/
8051	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8052	{
8053	Assert(TRPMHasTrap(pVCpu));
8054	Assert(!pVCpu->hm.s.Event.fPending);
8055
8056	uint8_t uVector;
8057	TRPMEVENT enmTrpmEvent;
8058	RTGCUINT uErrCode;
8059	RTGCUINTPTR GCPtrFaultAddress;
8060	uint8_t cbInstr;
8061
8062	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8063	AssertRC(rc);
8064
8065	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
8066	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
8067	if (enmTrpmEvent == TRPM_TRAP)
8068	{
8069	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
8070	* generated using INT1 (ICEBP). */
8071	switch (uVector)
8072	{
8073	case X86_XCPT_NMI:
8074	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8075	break;
8076
8077	case X86_XCPT_BP:
8078	case X86_XCPT_OF:
8079	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8080	break;
8081
8082	case X86_XCPT_PF:
8083	case X86_XCPT_DF:
8084	case X86_XCPT_TS:
8085	case X86_XCPT_NP:
8086	case X86_XCPT_SS:
8087	case X86_XCPT_GP:
8088	case X86_XCPT_AC:
8089	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
8090	RT_FALL_THRU();
8091	default:
8092	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8093	break;
8094	}
8095	}
8096	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
8097	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8098	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
8099	{
8100	switch (uVector)
8101	{
8102	case X86_XCPT_BP:
8103	case X86_XCPT_OF:
8104	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8105	break;
8106
8107	default:
8108	Assert(uVector == X86_XCPT_DB);
8109	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8110	break;
8111	}
8112	}
8113	else
8114	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
8115
8116	rc = TRPMResetTrap(pVCpu);
8117	AssertRC(rc);
8118	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8119	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8120
8121	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8122	}
8123
8124
8125	/**
8126	* Converts the pending HM event into a TRPM trap.
8127	*
8128	* @param pVCpu The cross context virtual CPU structure.
8129	*/
8130	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8131	{
8132	Assert(pVCpu->hm.s.Event.fPending);
8133
8134	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8135	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8136	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8137	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8138
8139	/* If a trap was already pending, we did something wrong! */
8140	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8141
8142	/** @todo Use HMVmxEventToTrpmEventType() later. */
8143	TRPMEVENT enmTrapType;
8144	switch (uVectorType)
8145	{
8146	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8147	enmTrapType = TRPM_HARDWARE_INT;
8148	break;
8149
8150	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8151	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8152	enmTrapType = TRPM_TRAP;
8153	break;
8154
8155	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8156	Assert(uVector == X86_XCPT_DB);
8157	enmTrapType = TRPM_SOFTWARE_INT;
8158	break;
8159
8160	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8161	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8162	enmTrapType = TRPM_SOFTWARE_INT;
8163	break;
8164
8165	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8166	enmTrapType = TRPM_SOFTWARE_INT;
8167	break;
8168
8169	default:
8170	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8171	enmTrapType = TRPM_32BIT_HACK;
8172	break;
8173	}
8174
8175	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8176
8177	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8178	AssertRC(rc);
8179
8180	if (fErrorCodeValid)
8181	TRPMSetErrorCode(pVCpu, uErrorCode);
8182
8183	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8184	&& uVector == X86_XCPT_PF)
8185	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8186	else if (enmTrapType == TRPM_SOFTWARE_INT)
8187	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8188
8189	/* We're now done converting the pending event. */
8190	pVCpu->hm.s.Event.fPending = false;
8191	}
8192
8193
8194	/**
8195	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8196	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8197	*
8198	* @param pVCpu The cross context virtual CPU structure.
8199	* @param pVmcsInfo The VMCS info. object.
8200	*/
8201	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8202	{
8203	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8204	{
8205	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8206	{
8207	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8208	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8209	AssertRC(rc);
8210	}
8211	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8212	}
8213
8214
8215	/**
8216	* Clears the interrupt-window exiting control in the VMCS.
8217	*
8218	* @param pVmcsInfo The VMCS info. object.
8219	*/
8220	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8221	{
8222	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8223	{
8224	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8225	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8226	}
8227	return VINF_SUCCESS;
8228	}
8229
8230
8231	/**
8232	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8233	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8234	*
8235	* @param pVCpu The cross context virtual CPU structure.
8236	* @param pVmcsInfo The VMCS info. object.
8237	*/
8238	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8239	{
8240	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8241	{
8242	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8243	{
8244	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8245	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8246	AssertRC(rc);
8247	Log4Func(("Setup NMI-window exiting\n"));
8248	}
8249	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8250	}
8251
8252
8253	/**
8254	* Clears the NMI-window exiting control in the VMCS.
8255	*
8256	* @param pVmcsInfo The VMCS info. object.
8257	*/
8258	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8259	{
8260	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8261	{
8262	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8263	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8264	}
8265	return VINF_SUCCESS;
8266	}
8267
8268
8269	/**
8270	* Does the necessary state syncing before returning to ring-3 for any reason
8271	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8272	*
8273	* @returns VBox status code.
8274	* @param pVCpu The cross context virtual CPU structure.
8275	* @param fImportState Whether to import the guest state from the VMCS back
8276	* to the guest-CPU context.
8277	*
8278	* @remarks No-long-jmp zone!!!
8279	*/
8280	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8281	{
8282	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8283	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8284
8285	RTCPUID idCpu = RTMpCpuId();
8286	Log4Func(("HostCpuId=%u\n", idCpu));
8287
8288	/*
8289	* !!! IMPORTANT !!!
8290	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8291	*/
8292
8293	/* Save the guest state if necessary. */
8294	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8295	if (fImportState)
8296	{
8297	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8298	AssertRCReturn(rc, rc);
8299	}
8300
8301	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8302	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8303	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8304
8305	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8306	#ifdef VBOX_STRICT
8307	if (CPUMIsHyperDebugStateActive(pVCpu))
8308	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8309	#endif
8310	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8311	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8312	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8313
8314	#if HC_ARCH_BITS == 64
8315	/* Restore host-state bits that VT-x only restores partially. */
8316	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8317	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8318	{
8319	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8320	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8321	}
8322	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8323	#endif
8324
8325	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8326	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8327	{
8328	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8329	if (!fImportState)
8330	{
8331	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8332	AssertRCReturn(rc, rc);
8333	}
8334	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8335	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8336	}
8337	else
8338	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8339
8340	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8341	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8342
8343	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8344	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8345	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8346	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8347	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8348	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8349	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8350	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8351	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8352
8353	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8354
8355	/** @todo This partially defeats the purpose of having preemption hooks.
8356	* The problem is, deregistering the hooks should be moved to a place that
8357	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8358	* context.
8359	*/
8360	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8361	AssertRCReturn(rc, rc);
8362
8363	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8364	NOREF(idCpu);
8365	return VINF_SUCCESS;
8366	}
8367
8368
8369	/**
8370	* Leaves the VT-x session.
8371	*
8372	* @returns VBox status code.
8373	* @param pVCpu The cross context virtual CPU structure.
8374	*
8375	* @remarks No-long-jmp zone!!!
8376	*/
8377	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8378	{
8379	HM_DISABLE_PREEMPT(pVCpu);
8380	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8381	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8382	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8383
8384	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8385	and done this from the VMXR0ThreadCtxCallback(). */
8386	if (!pVCpu->hm.s.fLeaveDone)
8387	{
8388	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8389	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8390	pVCpu->hm.s.fLeaveDone = true;
8391	}
8392	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8393
8394	/*
8395	* !!! IMPORTANT !!!
8396	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8397	*/
8398
8399	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8400	/** @todo Deregistering here means we need to VMCLEAR always
8401	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8402	* for calling VMMR0ThreadCtxHookDisable here! */
8403	VMMR0ThreadCtxHookDisable(pVCpu);
8404
8405	/* Leave HM context. This takes care of local init (term). */
8406	int rc = HMR0LeaveCpu(pVCpu);
8407
8408	HM_RESTORE_PREEMPT();
8409	return rc;
8410	}
8411
8412
8413	/**
8414	* Does the necessary state syncing before doing a longjmp to ring-3.
8415	*
8416	* @returns VBox status code.
8417	* @param pVCpu The cross context virtual CPU structure.
8418	*
8419	* @remarks No-long-jmp zone!!!
8420	*/
8421	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8422	{
8423	return hmR0VmxLeaveSession(pVCpu);
8424	}
8425
8426
8427	/**
8428	* Take necessary actions before going back to ring-3.
8429	*
8430	* An action requires us to go back to ring-3. This function does the necessary
8431	* steps before we can safely return to ring-3. This is not the same as longjmps
8432	* to ring-3, this is voluntary and prepares the guest so it may continue
8433	* executing outside HM (recompiler/IEM).
8434	*
8435	* @returns VBox status code.
8436	* @param pVCpu The cross context virtual CPU structure.
8437	* @param rcExit The reason for exiting to ring-3. Can be
8438	* VINF_VMM_UNKNOWN_RING3_CALL.
8439	*/
8440	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8441	{
8442	Assert(pVCpu);
8443	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8444
8445	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8446	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8447	{
8448	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8449	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8450	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8451	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8452	}
8453
8454	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8455	VMMRZCallRing3Disable(pVCpu);
8456	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8457
8458	/*
8459	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8460	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8461	*
8462	* This is because execution may continue from ring-3 and we would need to inject
8463	* the event from there (hence place it back in TRPM).
8464	*/
8465	if (pVCpu->hm.s.Event.fPending)
8466	{
8467	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8468	Assert(!pVCpu->hm.s.Event.fPending);
8469
8470	/* Clear the events from the VMCS. */
8471	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8472	AssertRCReturn(rc, rc);
8473	}
8474	#ifdef VBOX_STRICT
8475	else
8476	{
8477	/*
8478	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8479	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8480	* occasionally, see @bugref{9180#c42}.
8481	*/
8482	uint32_t uEntryIntInfo;
8483	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8484	AssertRC(rc);
8485	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8486	}
8487	#endif
8488
8489	/*
8490	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8491	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8492	* (e.g. TPR below threshold).
8493	*/
8494	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8495	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8496	AssertRCReturn(rc, rc);
8497
8498	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8499	and if we're injecting an event we should have a TRPM trap pending. */
8500	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8501	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8502	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8503	#endif
8504
8505	/* Save guest state and restore host state bits. */
8506	rc = hmR0VmxLeaveSession(pVCpu);
8507	AssertRCReturn(rc, rc);
8508	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8509
8510	/* Thread-context hooks are unregistered at this point!!! */
8511
8512	/* Sync recompiler state. */
8513	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8514	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8515	\| CPUM_CHANGED_LDTR
8516	\| CPUM_CHANGED_GDTR
8517	\| CPUM_CHANGED_IDTR
8518	\| CPUM_CHANGED_TR
8519	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8520	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8521	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8522	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8523
8524	Assert(!pVCpu->hm.s.fClearTrapFlag);
8525
8526	/* Update the exit-to-ring 3 reason. */
8527	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8528
8529	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8530	if ( rcExit != VINF_EM_RAW_INTERRUPT
8531	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8532	{
8533	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8534	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8535	}
8536
8537	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8538
8539	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8540	VMMRZCallRing3RemoveNotification(pVCpu);
8541	VMMRZCallRing3Enable(pVCpu);
8542
8543	return rc;
8544	}
8545
8546
8547	/**
8548	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8549	* longjump to ring-3 and possibly get preempted.
8550	*
8551	* @returns VBox status code.
8552	* @param pVCpu The cross context virtual CPU structure.
8553	* @param enmOperation The operation causing the ring-3 longjump.
8554	* @param pvUser User argument, currently unused, NULL.
8555	*/
8556	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8557	{
8558	RT_NOREF(pvUser);
8559	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8560	{
8561	/*
8562	* !!! IMPORTANT !!!
8563	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8564	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8565	*/
8566	VMMRZCallRing3RemoveNotification(pVCpu);
8567	VMMRZCallRing3Disable(pVCpu);
8568	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8569	RTThreadPreemptDisable(&PreemptState);
8570
8571	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8572	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8573	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8574	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8575
8576	#if HC_ARCH_BITS == 64
8577	/* Restore host-state bits that VT-x only restores partially. */
8578	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8579	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8580	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8581	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8582	#endif
8583
8584	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8585	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8586	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8587
8588	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8589	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8590	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8591
8592	/* Clear the current VMCS data back to memory. */
8593	hmR0VmxClearVmcs(pVmcsInfo);
8594
8595	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8596	VMMR0ThreadCtxHookDisable(pVCpu);
8597	HMR0LeaveCpu(pVCpu);
8598	RTThreadPreemptRestore(&PreemptState);
8599	return VINF_SUCCESS;
8600	}
8601
8602	Assert(pVCpu);
8603	Assert(pvUser);
8604	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8605	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8606
8607	VMMRZCallRing3Disable(pVCpu);
8608	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8609
8610	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8611
8612	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8613	AssertRCReturn(rc, rc);
8614
8615	VMMRZCallRing3Enable(pVCpu);
8616	return VINF_SUCCESS;
8617	}
8618
8619
8620	/**
8621	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8622	* stack.
8623	*
8624	* @returns Strict VBox status code (i.e. informational status codes too).
8625	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8626	* @param pVCpu The cross context virtual CPU structure.
8627	* @param uValue The value to push to the guest stack.
8628	*/
8629	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8630	{
8631	/*
8632	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8633	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8634	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8635	*/
8636	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8637	if (pCtx->sp == 1)
8638	return VINF_EM_RESET;
8639	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8640	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8641	AssertRC(rc);
8642	return rc;
8643	}
8644
8645
8646	/**
8647	* Injects an event into the guest upon VM-entry by updating the relevant fields
8648	* in the VM-entry area in the VMCS.
8649	*
8650	* @returns Strict VBox status code (i.e. informational status codes too).
8651	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8652	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8653	*
8654	* @param pVCpu The cross context virtual CPU structure.
8655	* @param pVmxTransient The VMX-transient structure.
8656	* @param pEvent The event being injected.
8657	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8658	* This will be updated if necessary. This cannot not
8659	* be NULL.
8660	* @param fStepping Whether we're single-stepping guest execution and
8661	* should return VINF_EM_DBG_STEPPED if the event is
8662	* injected directly (registers modified by us, not by
8663	* hardware on VM-entry).
8664	*/
8665	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8666	uint32_t *pfIntrState)
8667	{
8668	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8669	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8670	Assert(pfIntrState);
8671
8672	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8673	uint32_t u32IntInfo = pEvent->u64IntInfo;
8674	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8675	uint32_t const cbInstr = pEvent->cbInstr;
8676	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8677	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8678	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8679
8680	#ifdef VBOX_STRICT
8681	/*
8682	* Validate the error-code-valid bit for hardware exceptions.
8683	* No error codes for exceptions in real-mode.
8684	*
8685	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8686	*/
8687	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8688	&& !CPUMIsGuestInRealModeEx(pCtx))
8689	{
8690	switch (uVector)
8691	{
8692	case X86_XCPT_PF:
8693	case X86_XCPT_DF:
8694	case X86_XCPT_TS:
8695	case X86_XCPT_NP:
8696	case X86_XCPT_SS:
8697	case X86_XCPT_GP:
8698	case X86_XCPT_AC:
8699	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8700	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8701	RT_FALL_THRU();
8702	default:
8703	break;
8704	}
8705	}
8706
8707	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8708	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8709	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8710	#endif
8711
8712	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8713
8714	/*
8715	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8716	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8717	* interrupt handler in the (real-mode) guest.
8718	*
8719	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8720	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8721	*/
8722	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8723	{
8724	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8725	{
8726	/*
8727	* For CPUs with unrestricted guest execution enabled and with the guest
8728	* in real-mode, we must not set the deliver-error-code bit.
8729	*
8730	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8731	*/
8732	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8733	}
8734	else
8735	{
8736	PVM pVM = pVCpu->CTX_SUFF(pVM);
8737	Assert(PDMVmmDevHeapIsEnabled(pVM));
8738	Assert(pVM->hm.s.vmx.pRealModeTSS);
8739	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8740
8741	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8742	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8743	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8744	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8745	AssertRCReturn(rc2, rc2);
8746
8747	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8748	size_t const cbIdtEntry = sizeof(X86IDTR16);
8749	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8750	{
8751	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8752	if (uVector == X86_XCPT_DF)
8753	return VINF_EM_RESET;
8754
8755	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8756	No error codes for exceptions in real-mode. */
8757	if (uVector == X86_XCPT_GP)
8758	{
8759	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8760	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8761	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8762	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8763	HMEVENT EventXcptDf;
8764	RT_ZERO(EventXcptDf);
8765	EventXcptDf.u64IntInfo = uXcptDfInfo;
8766	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8767	}
8768
8769	/*
8770	* If we're injecting an event with no valid IDT entry, inject a #GP.
8771	* No error codes for exceptions in real-mode.
8772	*
8773	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8774	*/
8775	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8776	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8777	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8778	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8779	HMEVENT EventXcptGp;
8780	RT_ZERO(EventXcptGp);
8781	EventXcptGp.u64IntInfo = uXcptGpInfo;
8782	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8783	}
8784
8785	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8786	uint16_t uGuestIp = pCtx->ip;
8787	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8788	{
8789	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8790	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8791	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8792	}
8793	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8794	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8795
8796	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8797	X86IDTR16 IdtEntry;
8798	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8799	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8800	AssertRCReturn(rc2, rc2);
8801
8802	/* Construct the stack frame for the interrupt/exception handler. */
8803	VBOXSTRICTRC rcStrict;
8804	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8805	if (rcStrict == VINF_SUCCESS)
8806	{
8807	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8808	if (rcStrict == VINF_SUCCESS)
8809	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8810	}
8811
8812	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8813	if (rcStrict == VINF_SUCCESS)
8814	{
8815	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8816	pCtx->rip = IdtEntry.offSel;
8817	pCtx->cs.Sel = IdtEntry.uSel;
8818	pCtx->cs.ValidSel = IdtEntry.uSel;
8819	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8820	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8821	&& uVector == X86_XCPT_PF)
8822	pCtx->cr2 = GCPtrFault;
8823
8824	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8825	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8826	\| HM_CHANGED_GUEST_RSP);
8827
8828	/*
8829	* If we delivered a hardware exception (other than an NMI) and if there was
8830	* block-by-STI in effect, we should clear it.
8831	*/
8832	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8833	{
8834	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8835	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8836	Log4Func(("Clearing inhibition due to STI\n"));
8837	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8838	}
8839
8840	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8841	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8842
8843	/*
8844	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8845	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8846	*/
8847	pVCpu->hm.s.Event.fPending = false;
8848
8849	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8850	if (fStepping)
8851	rcStrict = VINF_EM_DBG_STEPPED;
8852	}
8853	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8854	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8855	return rcStrict;
8856	}
8857	}
8858
8859	/*
8860	* Validate.
8861	*/
8862	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8863	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8864
8865	/*
8866	* Inject the event into the VMCS.
8867	*/
8868	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8869	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8870	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8871	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8872	AssertRCReturn(rc, rc);
8873
8874	/*
8875	* Update guest CR2 if this is a page-fault.
8876	*/
8877	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8878	&& uVector == X86_XCPT_PF)
8879	pCtx->cr2 = GCPtrFault;
8880
8881	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8882	return VINF_SUCCESS;
8883	}
8884
8885
8886	/**
8887	* Evaluates the event to be delivered to the guest and sets it as the pending
8888	* event.
8889	*
8890	* @returns Strict VBox status code (i.e. informational status codes too).
8891	* @param pVCpu The cross context virtual CPU structure.
8892	* @param pVmxTransient The VMX-transient structure.
8893	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8894	*/
8895	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8896	{
8897	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8898	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8899
8900	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8901	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8902	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8903	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8904	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8905
8906	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8907	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8908	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8909	Assert(!TRPMHasTrap(pVCpu));
8910	Assert(pfIntrState);
8911
8912	*pfIntrState = fIntrState;
8913
8914	/*
8915	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8916	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8917	*/
8918	/** @todo SMI. SMIs take priority over NMIs. */
8919	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8920	{
8921	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8922	if ( !pVCpu->hm.s.Event.fPending
8923	&& !fBlockNmi
8924	&& !fBlockSti
8925	&& !fBlockMovSS)
8926	{
8927	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8928	if ( pVmxTransient->fIsNestedGuest
8929	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8930	return IEMExecVmxVmexitXcptNmi(pVCpu);
8931	#endif
8932	hmR0VmxSetPendingXcptNmi(pVCpu);
8933	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8934	Log4Func(("Pending NMI\n"));
8935	}
8936	else
8937	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8938	}
8939	/*
8940	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8941	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8942	*/
8943	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8944	&& !pVCpu->hm.s.fSingleInstruction)
8945	{
8946	Assert(!DBGFIsStepping(pVCpu));
8947	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8948	AssertRCReturn(rc, rc);
8949	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8950	if ( !pVCpu->hm.s.Event.fPending
8951	&& !fBlockInt
8952	&& !fBlockSti
8953	&& !fBlockMovSS)
8954	{
8955	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8956	if ( pVmxTransient->fIsNestedGuest
8957	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8958	{
8959	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8960	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8961	return rcStrict;
8962	}
8963	#endif
8964	uint8_t u8Interrupt;
8965	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8966	if (RT_SUCCESS(rc))
8967	{
8968	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8969	if ( pVmxTransient->fIsNestedGuest
8970	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8971	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8972	{
8973	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8974	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8975	return rcStrict;
8976	}
8977	#endif
8978	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8979	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8980	}
8981	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8982	{
8983	if ( !pVmxTransient->fIsNestedGuest
8984	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8985	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8986	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8987
8988	/*
8989	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8990	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8991	* need to re-set this force-flag here.
8992	*/
8993	}
8994	else
8995	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8996	}
8997	else
8998	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8999	}
9000
9001	return VINF_SUCCESS;
9002	}
9003
9004
9005	/**
9006	* Injects any pending events into the guest if the guest is in a state to
9007	* receive them.
9008	*
9009	* @returns Strict VBox status code (i.e. informational status codes too).
9010	* @param pVCpu The cross context virtual CPU structure.
9011	* @param pVmxTransient The VMX-transient structure.
9012	* @param fIntrState The VT-x guest-interruptibility state.
9013	* @param fStepping Whether we are single-stepping the guest using the
9014	* hypervisor debugger and should return
9015	* VINF_EM_DBG_STEPPED if the event was dispatched
9016	* directly.
9017	*/
9018	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9019	{
9020	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9021	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9022
9023	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9024	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9025
9026	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9027	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9028	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9029	Assert(!TRPMHasTrap(pVCpu));
9030
9031	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9032	if (pVCpu->hm.s.Event.fPending)
9033	{
9034	/*
9035	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9036	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9037	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9038	*
9039	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9040	*/
9041	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9042	#ifdef VBOX_STRICT
9043	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9044	{
9045	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9046	Assert(!fBlockInt);
9047	Assert(!fBlockSti);
9048	Assert(!fBlockMovSS);
9049	}
9050	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9051	{
9052	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9053	Assert(!fBlockSti);
9054	Assert(!fBlockMovSS);
9055	Assert(!fBlockNmi);
9056	}
9057	#endif
9058	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9059	uIntType));
9060
9061	/*
9062	* Inject the event and get any changes to the guest-interruptibility state.
9063	*
9064	* The guest-interruptibility state may need to be updated if we inject the event
9065	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9066	*/
9067	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9068	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9069
9070	/*
9071	* If we are executing a nested-guest make sure that we should intercept subsequent
9072	* events. The one we are injecting might be part of VM-entry.
9073	*/
9074	if (pVmxTransient->fIsNestedGuest)
9075	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
9076
9077	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9078	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9079	else
9080	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9081	}
9082
9083	/*
9084	* Update the guest-interruptibility state.
9085	*
9086	* This is required for the real-on-v86 software interrupt injection case above, as well as
9087	* updates to the guest state from ring-3 or IEM/REM.
9088	*/
9089	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9090	AssertRCReturn(rc, rc);
9091
9092	/*
9093	* There's no need to clear the VM-entry interruption-information field here if we're not
9094	* injecting anything. VT-x clears the valid bit on every VM-exit.
9095	*
9096	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9097	*/
9098
9099	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9100	NOREF(fBlockMovSS); NOREF(fBlockSti);
9101	return rcStrict;
9102	}
9103
9104
9105	/**
9106	* Enters the VT-x session.
9107	*
9108	* @returns VBox status code.
9109	* @param pVCpu The cross context virtual CPU structure.
9110	*/
9111	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9112	{
9113	AssertPtr(pVCpu);
9114	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9115	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9116
9117	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9118	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9119	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9120
9121	#ifdef VBOX_STRICT
9122	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9123	RTCCUINTREG uHostCR4 = ASMGetCR4();
9124	if (!(uHostCR4 & X86_CR4_VMXE))
9125	{
9126	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9127	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9128	}
9129	#endif
9130
9131	/*
9132	* Load the appropriate VMCS as the current and active one.
9133	*/
9134	PVMXVMCSINFO pVmcsInfo;
9135	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9136	if (!fInNestedGuestMode)
9137	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9138	else
9139	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9140	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9141	if (RT_SUCCESS(rc))
9142	{
9143	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9144	pVCpu->hm.s.fLeaveDone = false;
9145	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9146
9147	/*
9148	* Do the EMT scheduled L1D flush here if needed.
9149	*/
9150	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9151	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9152	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9153	hmR0MdsClear();
9154	}
9155	return rc;
9156	}
9157
9158
9159	/**
9160	* The thread-context callback (only on platforms which support it).
9161	*
9162	* @param enmEvent The thread-context event.
9163	* @param pVCpu The cross context virtual CPU structure.
9164	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9165	* @thread EMT(pVCpu)
9166	*/
9167	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9168	{
9169	NOREF(fGlobalInit);
9170
9171	switch (enmEvent)
9172	{
9173	case RTTHREADCTXEVENT_OUT:
9174	{
9175	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9176	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9177	VMCPU_ASSERT_EMT(pVCpu);
9178
9179	/* No longjmps (logger flushes, locks) in this fragile context. */
9180	VMMRZCallRing3Disable(pVCpu);
9181	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9182
9183	/* Restore host-state (FPU, debug etc.) */
9184	if (!pVCpu->hm.s.fLeaveDone)
9185	{
9186	/*
9187	* Do -not- import the guest-state here as we might already be in the middle of importing
9188	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9189	*/
9190	hmR0VmxLeave(pVCpu, false /* fImportState */);
9191	pVCpu->hm.s.fLeaveDone = true;
9192	}
9193
9194	/* Leave HM context, takes care of local init (term). */
9195	int rc = HMR0LeaveCpu(pVCpu);
9196	AssertRC(rc);
9197
9198	/* Restore longjmp state. */
9199	VMMRZCallRing3Enable(pVCpu);
9200	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9201	break;
9202	}
9203
9204	case RTTHREADCTXEVENT_IN:
9205	{
9206	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9207	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9208	VMCPU_ASSERT_EMT(pVCpu);
9209
9210	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9211	VMMRZCallRing3Disable(pVCpu);
9212	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9213
9214	/* Initialize the bare minimum state required for HM. This takes care of
9215	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9216	int rc = hmR0EnterCpu(pVCpu);
9217	AssertRC(rc);
9218	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9219	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9220
9221	/* Load the active VMCS as the current one. */
9222	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9223	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9224	AssertRC(rc);
9225	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9226	pVCpu->hm.s.fLeaveDone = false;
9227
9228	/* Do the EMT scheduled L1D flush if needed. */
9229	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9230	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9231
9232	/* Restore longjmp state. */
9233	VMMRZCallRing3Enable(pVCpu);
9234	break;
9235	}
9236
9237	default:
9238	break;
9239	}
9240	}
9241
9242
9243	/**
9244	* Exports the host state into the VMCS host-state area.
9245	* Sets up the VM-exit MSR-load area.
9246	*
9247	* The CPU state will be loaded from these fields on every successful VM-exit.
9248	*
9249	* @returns VBox status code.
9250	* @param pVCpu The cross context virtual CPU structure.
9251	*
9252	* @remarks No-long-jump zone!!!
9253	*/
9254	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9255	{
9256	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9257
9258	int rc = VINF_SUCCESS;
9259	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9260	{
9261	rc = hmR0VmxExportHostControlRegs();
9262	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9263
9264	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9265	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9266
9267	rc = hmR0VmxExportHostMsrs(pVCpu);
9268	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9269
9270	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9271	}
9272	return rc;
9273	}
9274
9275
9276	/**
9277	* Saves the host state in the VMCS host-state.
9278	*
9279	* @returns VBox status code.
9280	* @param pVCpu The cross context virtual CPU structure.
9281	*
9282	* @remarks No-long-jump zone!!!
9283	*/
9284	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9285	{
9286	AssertPtr(pVCpu);
9287	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9288
9289	/*
9290	* Export the host state here while entering HM context.
9291	* When thread-context hooks are used, we might get preempted and have to re-save the host
9292	* state but most of the time we won't be, so do it here before we disable interrupts.
9293	*/
9294	return hmR0VmxExportHostState(pVCpu);
9295	}
9296
9297
9298	/**
9299	* Exports the guest state into the VMCS guest-state area.
9300	*
9301	* The will typically be done before VM-entry when the guest-CPU state and the
9302	* VMCS state may potentially be out of sync.
9303	*
9304	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9305	* VM-entry controls.
9306	* Sets up the appropriate VMX non-root function to execute guest code based on
9307	* the guest CPU mode.
9308	*
9309	* @returns VBox strict status code.
9310	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9311	* without unrestricted guest execution and the VMMDev is not presently
9312	* mapped (e.g. EFI32).
9313	*
9314	* @param pVCpu The cross context virtual CPU structure.
9315	* @param pVmxTransient The VMX-transient structure.
9316	*
9317	* @remarks No-long-jump zone!!!
9318	*/
9319	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9320	{
9321	AssertPtr(pVCpu);
9322	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9323	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9324
9325	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9326
9327	/*
9328	* Determine real-on-v86 mode.
9329	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9330	*/
9331	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9332	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9333	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9334	pVmcsInfo->RealMode. fRealOnV86Active = false;
9335	else
9336	{
9337	Assert(!pVmxTransient->fIsNestedGuest);
9338	pVmcsInfo->RealMode.fRealOnV86Active = true;
9339	}
9340
9341	/*
9342	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9343	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9344	*/
9345	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9346	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9347	* be a need to evaluate this everytime since I'm pretty sure we intercept
9348	* all guest paging mode changes. */
9349	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9350	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9351
9352	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9353	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9354
9355	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9356	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9357
9358	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9359	if (rcStrict == VINF_SUCCESS)
9360	{ /* likely */ }
9361	else
9362	{
9363	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9364	return rcStrict;
9365	}
9366
9367	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9368	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9369
9370	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9371	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9372
9373	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9374	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9375
9376	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9377	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9378
9379	rc = hmR0VmxExportGuestRip(pVCpu);
9380	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9381	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9382	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9383
9384	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9385	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9386	\| HM_CHANGED_GUEST_CR2
9387	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9388	\| HM_CHANGED_GUEST_X87
9389	\| HM_CHANGED_GUEST_SSE_AVX
9390	\| HM_CHANGED_GUEST_OTHER_XSAVE
9391	\| HM_CHANGED_GUEST_XCRx
9392	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9393	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9394	\| HM_CHANGED_GUEST_TSC_AUX
9395	\| HM_CHANGED_GUEST_OTHER_MSRS
9396	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9397	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9398
9399	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9400	return rc;
9401	}
9402
9403
9404	/**
9405	* Exports the state shared between the host and guest into the VMCS.
9406	*
9407	* @param pVCpu The cross context virtual CPU structure.
9408	* @param pVmxTransient The VMX-transient structure.
9409	*
9410	* @remarks No-long-jump zone!!!
9411	*/
9412	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9413	{
9414	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9415	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9416
9417	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9418	{
9419	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9420	AssertRC(rc);
9421	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9422
9423	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9424	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9425	{
9426	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9427	AssertRC(rc);
9428	}
9429	}
9430
9431	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9432	{
9433	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9434	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9435	}
9436
9437	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9438	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9439	}
9440
9441
9442	/**
9443	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9444	*
9445	* @returns Strict VBox status code (i.e. informational status codes too).
9446	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9447	* without unrestricted guest execution and the VMMDev is not presently
9448	* mapped (e.g. EFI32).
9449	*
9450	* @param pVCpu The cross context virtual CPU structure.
9451	* @param pVmxTransient The VMX-transient structure.
9452	*
9453	* @remarks No-long-jump zone!!!
9454	*/
9455	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9456	{
9457	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9458	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9459	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9460
9461	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9462	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9463	#endif
9464
9465	/*
9466	* For many exits it's only RIP that changes and hence try to export it first
9467	* without going through a lot of change flag checks.
9468	*/
9469	VBOXSTRICTRC rcStrict;
9470	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9471	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9472	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9473	{
9474	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9475	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9476	{ /* likely */}
9477	else
9478	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9480	}
9481	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9482	{
9483	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9484	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9485	{ /* likely */}
9486	else
9487	{
9488	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9489	VBOXSTRICTRC_VAL(rcStrict)));
9490	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9491	return rcStrict;
9492	}
9493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9494	}
9495	else
9496	rcStrict = VINF_SUCCESS;
9497
9498	#ifdef VBOX_STRICT
9499	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9500	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9501	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9502	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9503	("fCtxChanged=%#RX64\n", fCtxChanged));
9504	#endif
9505	return rcStrict;
9506	}
9507
9508
9509	/**
9510	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9511	* and update error record fields accordingly.
9512	*
9513	* @return VMX_IGS_* return codes.
9514	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9515	* wrong with the guest state.
9516	*
9517	* @param pVCpu The cross context virtual CPU structure.
9518	* @param pVmcsInfo The VMCS info. object.
9519	*
9520	* @remarks This function assumes our cache of the VMCS controls
9521	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9522	*/
9523	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9524	{
9525	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9526	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9527	uError = (err); \
9528	break; \
9529	} else do { } while (0)
9530
9531	int rc;
9532	PVM pVM = pVCpu->CTX_SUFF(pVM);
9533	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9534	uint32_t uError = VMX_IGS_ERROR;
9535	uint32_t u32Val;
9536	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9537
9538	do
9539	{
9540	/*
9541	* CR0.
9542	*/
9543	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9544	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9545	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9546	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9547	if (fUnrestrictedGuest)
9548	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9549
9550	uint32_t u32GuestCr0;
9551	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9552	AssertRCBreak(rc);
9553	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9554	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9555	if ( !fUnrestrictedGuest
9556	&& (u32GuestCr0 & X86_CR0_PG)
9557	&& !(u32GuestCr0 & X86_CR0_PE))
9558	{
9559	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9560	}
9561
9562	/*
9563	* CR4.
9564	*/
9565	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9566	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9567
9568	uint32_t u32GuestCr4;
9569	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9570	AssertRCBreak(rc);
9571	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9572	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9573
9574	/*
9575	* IA32_DEBUGCTL MSR.
9576	*/
9577	uint64_t u64Val;
9578	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9579	AssertRCBreak(rc);
9580	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9581	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9582	{
9583	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9584	}
9585	uint64_t u64DebugCtlMsr = u64Val;
9586
9587	#ifdef VBOX_STRICT
9588	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9589	AssertRCBreak(rc);
9590	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9591	#endif
9592	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9593
9594	/*
9595	* RIP and RFLAGS.
9596	*/
9597	uint32_t u32Eflags;
9598	#if HC_ARCH_BITS == 64
9599	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9600	AssertRCBreak(rc);
9601	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9602	if ( !fLongModeGuest
9603	\|\| !pCtx->cs.Attr.n.u1Long)
9604	{
9605	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9606	}
9607	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9608	* must be identical if the "IA-32e mode guest" VM-entry
9609	* control is 1 and CS.L is 1. No check applies if the
9610	* CPU supports 64 linear-address bits. */
9611
9612	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9613	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9614	AssertRCBreak(rc);
9615	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9616	VMX_IGS_RFLAGS_RESERVED);
9617	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9618	u32Eflags = u64Val;
9619	#else
9620	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9621	AssertRCBreak(rc);
9622	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9623	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9624	#endif
9625
9626	if ( fLongModeGuest
9627	\|\| ( fUnrestrictedGuest
9628	&& !(u32GuestCr0 & X86_CR0_PE)))
9629	{
9630	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9631	}
9632
9633	uint32_t u32EntryInfo;
9634	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9635	AssertRCBreak(rc);
9636	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9637	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9638	{
9639	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9640	}
9641
9642	/*
9643	* 64-bit checks.
9644	*/
9645	#if HC_ARCH_BITS == 64
9646	if (fLongModeGuest)
9647	{
9648	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9649	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9650	}
9651
9652	if ( !fLongModeGuest
9653	&& (u32GuestCr4 & X86_CR4_PCIDE))
9654	{
9655	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9656	}
9657
9658	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9659	* 51:32 beyond the processor's physical-address width are 0. */
9660
9661	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9662	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9663	{
9664	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9665	}
9666
9667	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9668	AssertRCBreak(rc);
9669	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9670
9671	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9672	AssertRCBreak(rc);
9673	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9674	#endif
9675
9676	/*
9677	* PERF_GLOBAL MSR.
9678	*/
9679	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9680	{
9681	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9682	AssertRCBreak(rc);
9683	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9684	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9685	}
9686
9687	/*
9688	* PAT MSR.
9689	*/
9690	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9691	{
9692	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9693	AssertRCBreak(rc);
9694	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9695	for (unsigned i = 0; i < 8; i++)
9696	{
9697	uint8_t u8Val = (u64Val & 0xff);
9698	if ( u8Val != 0 /* UC */
9699	&& u8Val != 1 /* WC */
9700	&& u8Val != 4 /* WT */
9701	&& u8Val != 5 /* WP */
9702	&& u8Val != 6 /* WB */
9703	&& u8Val != 7 /* UC- */)
9704	{
9705	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9706	}
9707	u64Val >>= 8;
9708	}
9709	}
9710
9711	/*
9712	* EFER MSR.
9713	*/
9714	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9715	{
9716	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9717	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9718	AssertRCBreak(rc);
9719	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9720	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9721	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9722	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9723	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9724	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9725	* iemVmxVmentryCheckGuestState(). */
9726	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9727	\|\| !(u32GuestCr0 & X86_CR0_PG)
9728	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9729	VMX_IGS_EFER_LMA_LME_MISMATCH);
9730	}
9731
9732	/*
9733	* Segment registers.
9734	*/
9735	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9736	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9737	if (!(u32Eflags & X86_EFL_VM))
9738	{
9739	/* CS */
9740	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9741	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9742	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9743	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9744	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9745	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9746	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9747	/* CS cannot be loaded with NULL in protected mode. */
9748	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9749	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9750	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9751	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9752	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9753	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9754	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9755	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9756	else
9757	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9758
9759	/* SS */
9760	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9761	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9762	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9763	if ( !(pCtx->cr0 & X86_CR0_PE)
9764	\|\| pCtx->cs.Attr.n.u4Type == 3)
9765	{
9766	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9767	}
9768	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9769	{
9770	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9771	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9772	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9773	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9774	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9775	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9776	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9777	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9778	}
9779
9780	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9781	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9782	{
9783	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9784	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9785	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9786	\|\| pCtx->ds.Attr.n.u4Type > 11
9787	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9788	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9789	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9790	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9791	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9792	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9793	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9794	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9795	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9796	}
9797	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9798	{
9799	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9800	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9801	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9802	\|\| pCtx->es.Attr.n.u4Type > 11
9803	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9804	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9805	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9806	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9807	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9808	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9809	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9810	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9811	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9812	}
9813	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9814	{
9815	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9816	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9817	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9818	\|\| pCtx->fs.Attr.n.u4Type > 11
9819	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9820	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9821	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9822	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9823	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9824	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9825	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9826	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9827	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9828	}
9829	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9830	{
9831	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9832	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9833	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9834	\|\| pCtx->gs.Attr.n.u4Type > 11
9835	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9836	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9837	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9838	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9839	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9840	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9841	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9842	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9843	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9844	}
9845	/* 64-bit capable CPUs. */
9846	#if HC_ARCH_BITS == 64
9847	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9848	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9849	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9850	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9851	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9852	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9853	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9854	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9855	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9856	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9857	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9858	#endif
9859	}
9860	else
9861	{
9862	/* V86 mode checks. */
9863	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9864	if (pVmcsInfo->RealMode.fRealOnV86Active)
9865	{
9866	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9867	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9868	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9869	}
9870	else
9871	{
9872	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9873	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9874	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9875	}
9876
9877	/* CS */
9878	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9879	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9880	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9881	/* SS */
9882	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9883	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9884	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9885	/* DS */
9886	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9887	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9888	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9889	/* ES */
9890	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9891	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9892	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9893	/* FS */
9894	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9895	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9896	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9897	/* GS */
9898	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9899	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9900	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9901	/* 64-bit capable CPUs. */
9902	#if HC_ARCH_BITS == 64
9903	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9904	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9905	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9906	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9907	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9908	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9909	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9910	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9911	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9912	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9913	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9914	#endif
9915	}
9916
9917	/*
9918	* TR.
9919	*/
9920	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9921	/* 64-bit capable CPUs. */
9922	#if HC_ARCH_BITS == 64
9923	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9924	#endif
9925	if (fLongModeGuest)
9926	{
9927	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9928	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9929	}
9930	else
9931	{
9932	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9933	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9934	VMX_IGS_TR_ATTR_TYPE_INVALID);
9935	}
9936	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9937	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9938	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9939	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9940	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9941	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9942	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9943	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9944
9945	/*
9946	* GDTR and IDTR.
9947	*/
9948	#if HC_ARCH_BITS == 64
9949	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9950	AssertRCBreak(rc);
9951	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9952
9953	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9954	AssertRCBreak(rc);
9955	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9956	#endif
9957
9958	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9959	AssertRCBreak(rc);
9960	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9961
9962	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9963	AssertRCBreak(rc);
9964	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9965
9966	/*
9967	* Guest Non-Register State.
9968	*/
9969	/* Activity State. */
9970	uint32_t u32ActivityState;
9971	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9972	AssertRCBreak(rc);
9973	HMVMX_CHECK_BREAK( !u32ActivityState
9974	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9975	VMX_IGS_ACTIVITY_STATE_INVALID);
9976	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9977	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9978	uint32_t u32IntrState;
9979	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9980	AssertRCBreak(rc);
9981	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9982	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9983	{
9984	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9985	}
9986
9987	/** @todo Activity state and injecting interrupts. Left as a todo since we
9988	* currently don't use activity states but ACTIVE. */
9989
9990	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9991	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9992
9993	/* Guest interruptibility-state. */
9994	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9995	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9996	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9997	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9998	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9999	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10000	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10001	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10002	{
10003	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10004	{
10005	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10006	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10007	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10008	}
10009	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10010	{
10011	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10012	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10013	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10014	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10015	}
10016	}
10017	/** @todo Assumes the processor is not in SMM. */
10018	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10019	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10020	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10021	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10022	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10023	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10024	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10025	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10026	{
10027	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10028	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10029	}
10030
10031	/* Pending debug exceptions. */
10032	#if HC_ARCH_BITS == 64
10033	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10034	AssertRCBreak(rc);
10035	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10036	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10037	u32Val = u64Val; /* For pending debug exceptions checks below. */
10038	#else
10039	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10040	AssertRCBreak(rc);
10041	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10042	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10043	#endif
10044
10045	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10046	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10047	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10048	{
10049	if ( (u32Eflags & X86_EFL_TF)
10050	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10051	{
10052	/* Bit 14 is PendingDebug.BS. */
10053	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10054	}
10055	if ( !(u32Eflags & X86_EFL_TF)
10056	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10057	{
10058	/* Bit 14 is PendingDebug.BS. */
10059	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10060	}
10061	}
10062
10063	/* VMCS link pointer. */
10064	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10065	AssertRCBreak(rc);
10066	if (u64Val != UINT64_C(0xffffffffffffffff))
10067	{
10068	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10069	/** @todo Bits beyond the processor's physical-address width MBZ. */
10070	/** @todo 32-bit located in memory referenced by value of this field (as a
10071	* physical address) must contain the processor's VMCS revision ID. */
10072	/** @todo SMM checks. */
10073	}
10074
10075	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10076	* not using nested paging? */
10077	if ( pVM->hm.s.fNestedPaging
10078	&& !fLongModeGuest
10079	&& CPUMIsGuestInPAEModeEx(pCtx))
10080	{
10081	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10082	AssertRCBreak(rc);
10083	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10084
10085	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10086	AssertRCBreak(rc);
10087	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10088
10089	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10090	AssertRCBreak(rc);
10091	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10092
10093	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10094	AssertRCBreak(rc);
10095	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10096	}
10097
10098	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10099	if (uError == VMX_IGS_ERROR)
10100	uError = VMX_IGS_REASON_NOT_FOUND;
10101	} while (0);
10102
10103	pVCpu->hm.s.u32HMError = uError;
10104	return uError;
10105
10106	#undef HMVMX_ERROR_BREAK
10107	#undef HMVMX_CHECK_BREAK
10108	}
10109
10110
10111	/**
10112	* Setup the APIC-access page for virtualizing APIC access.
10113	*
10114	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
10115	* this not done as part of exporting guest state, see @bugref{8721}.
10116	*
10117	* @returns VBox status code.
10118	* @param pVCpu The cross context virtual CPU structure.
10119	*/
10120	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10121	{
10122	PVM pVM = pVCpu->CTX_SUFF(pVM);
10123	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10124
10125	Assert(PDMHasApic(pVM));
10126	Assert(u64MsrApicBase);
10127
10128	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10129	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10130
10131	/* Unalias any existing mapping. */
10132	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10133	AssertRCReturn(rc, rc);
10134
10135	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10136	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10137	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10138	AssertRCReturn(rc, rc);
10139
10140	/* Update the per-VCPU cache of the APIC base MSR. */
10141	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10142	return VINF_SUCCESS;
10143	}
10144
10145
10146	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10147	/**
10148	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10149	* nested-guest using hardware-assisted VMX.
10150	*
10151	* @param pVCpu The cross context virtual CPU structure.
10152	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10153	* @param pVmcsInfoGst The guest VMCS info. object.
10154	*/
10155	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10156	{
10157	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10158	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10159	Assert(pu64MsrBitmap);
10160
10161	/*
10162	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10163	* MSR that is intercepted by the guest is also intercepted while executing the
10164	* nested-guest using hardware-assisted VMX.
10165	*
10166	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10167	* nested-guest VM-exit even if the outer guest is not intercepting some
10168	* MSRs. We cannot assume the caller has initialized the nested-guest
10169	* MSR bitmap in this case.
10170	*
10171	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10172	* each VM-entry, hence initializing it once per-VM while setting up the
10173	* nested-guest VMCS is not sufficient.
10174	*/
10175	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10176	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10177	{
10178	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10179	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10180	Assert(pu64MsrBitmapNstGst);
10181	Assert(pu64MsrBitmapGst);
10182
10183	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10184	for (uint32_t i = 0; i < cFrags; i++)
10185	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10186	}
10187	else
10188	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10189	}
10190
10191
10192	/**
10193	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10194	* hardware-assisted VMX execution of the nested-guest.
10195	*
10196	* For a guest, we don't modify these controls once we set up the VMCS and hence
10197	* this function is never called.
10198	*
10199	* For nested-guests since the guest hypervisor provides these controls on every
10200	* nested-guest VM-entry and could potentially change them everytime we need to
10201	* merge them before every nested-guest VM-entry.
10202	*
10203	* @returns VBox status code.
10204	* @param pVCpu The cross context virtual CPU structure.
10205	*/
10206	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10207	{
10208	PVM pVM = pVCpu->CTX_SUFF(pVM);
10209	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10210	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10211	Assert(pVmcsNstGst);
10212
10213	/*
10214	* Merge the controls with the requirements of the guest VMCS.
10215	*
10216	* We do not need to validate the nested-guest VMX features specified in the
10217	* nested-guest VMCS with the features supported by the physical CPU as it's
10218	* already done by the VMLAUNCH/VMRESUME instruction emulation.
10219	*
10220	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
10221	* guest are derived from the VMX features supported by the physical CPU.
10222	*/
10223
10224	/* Pin-based VM-execution controls. */
10225	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10226
10227	/* Processor-based VM-execution controls. */
10228	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10229	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10230	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10231	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10232	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10233
10234	/* Secondary processor-based VM-execution controls. */
10235	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10236	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10237	\| VMX_PROC_CTLS2_INVPCID
10238	\| VMX_PROC_CTLS2_RDTSCP
10239	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10240	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10241	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10242	\| VMX_PROC_CTLS2_VMFUNC));
10243
10244	/*
10245	* VM-entry controls:
10246	* These controls contains state that depends on the nested-guest state (primarily
10247	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10248	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
10249	* properly continue executing the nested-guest if the EFER MSR changes but does not
10250	* cause a nested-guest VM-exits.
10251	*
10252	* VM-exit controls:
10253	* These controls specify the host state on return. We cannot use the controls from
10254	* the guest-hypervisor state as is as it would contain the guest state rather than
10255	* the host state. Since the host state is subject to change (e.g. preemption, trips
10256	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10257	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10258	*
10259	* VM-entry MSR-load:
10260	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
10261	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
10262	*
10263	* VM-exit MSR-store:
10264	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
10265	* context back into the VM-exit MSR-store area.
10266	*
10267	* VM-exit MSR-load areas:
10268	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
10269	* we can entirely ignore what the nested-hypervisor wants to load here.
10270	*/
10271
10272	/*
10273	* Exception bitmap.
10274	*
10275	* We could remove #UD from the guest bitmap and merge it with the nested-guest
10276	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
10277	* keep the code more flexible if intercepting exceptions become more dynamic in
10278	* the future we do it as part of exporting the nested-guest state.
10279	*/
10280	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10281
10282	/*
10283	* CR0/CR4 guest/host mask.
10284	*
10285	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
10286	* must cause VM-exits, so we need to merge them here.
10287	*/
10288	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10289	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10290
10291	/*
10292	* Page-fault error-code mask and match.
10293	*
10294	* Although we require unrestricted guest execution (and thereby nested-paging) for
10295	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10296	* normally intercept #PFs, it might intercept them for debugging purposes.
10297	*
10298	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
10299	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10300	*/
10301	uint32_t u32XcptPFMask;
10302	uint32_t u32XcptPFMatch;
10303	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10304	{
10305	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10306	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10307	}
10308	else
10309	{
10310	u32XcptPFMask = 0;
10311	u32XcptPFMatch = 0;
10312	}
10313
10314	/*
10315	* Pause-Loop exiting.
10316	*/
10317	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10318	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10319
10320	/*
10321	* I/O Bitmap.
10322	*
10323	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10324	* always intercept all I/O port accesses.
10325	*/
10326	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10327
10328	/*
10329	* APIC-access page.
10330	*
10331	* The APIC-access page address has already been initialized while setting up the
10332	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10333	* should not be on any consequence to the host or to the guest for that matter, but
10334	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10335	* emulation to keep it simple.
10336	*/
10337
10338	/*
10339	* Virtual-APIC page and TPR threshold.
10340	*
10341	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10342	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10343	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10344	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10345	*/
10346	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10347	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10348	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10349	{
10350	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10351
10352	/*
10353	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10354	* we would fail to obtain a valid host-physical address for its guest-physical
10355	* address.
10356	*
10357	* We currently do not support this scenario. Maybe in the future if there is a
10358	* pressing need we can explore making this particular set of conditions work.
10359	* Right now we just cause a VM-entry failure.
10360	*
10361	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10362	* so should not really failure at the moment.
10363	*/
10364	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10365	}
10366	else
10367	{
10368	/*
10369	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10370	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10371	* be taken care of by EPT/shadow paging.
10372	*/
10373	if (pVM->hm.s.fAllow64BitGuests)
10374	{
10375	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10376	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10377	}
10378	}
10379
10380	/*
10381	* Validate basic assumptions.
10382	*/
10383	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10384	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10385	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10386	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10387
10388	/*
10389	* Commit it to the nested-guest VMCS.
10390	*/
10391	int rc = VINF_SUCCESS;
10392	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10393	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10394	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10395	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10396	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10397	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10398	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10399	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10400	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10401	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10402	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10403	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10404	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10405	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10406	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10407	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10408	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10409	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10410	{
10411	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10412	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10413	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10414	}
10415	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10416	{
10417	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10418	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10419	}
10420	AssertRCReturn(rc, rc);
10421
10422	/*
10423	* Update the nested-guest VMCS cache.
10424	*/
10425	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10426	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10427	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10428	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10429	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10430	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10431	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10432	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10433	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10434
10435	/*
10436	* MSR bitmap.
10437	*
10438	* The MSR bitmap address has already been initialized while setting up the
10439	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10440	*/
10441	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10442	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10443
10444	return VINF_SUCCESS;
10445	}
10446	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10447
10448
10449	/**
10450	* Does the preparations before executing guest code in VT-x.
10451	*
10452	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10453	* recompiler/IEM. We must be cautious what we do here regarding committing
10454	* guest-state information into the VMCS assuming we assuredly execute the
10455	* guest in VT-x mode.
10456	*
10457	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10458	* the common-state (TRPM/forceflags), we must undo those changes so that the
10459	* recompiler/IEM can (and should) use them when it resumes guest execution.
10460	* Otherwise such operations must be done when we can no longer exit to ring-3.
10461	*
10462	* @returns Strict VBox status code (i.e. informational status codes too).
10463	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10464	* have been disabled.
10465	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10466	* double-fault into the guest.
10467	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10468	* dispatched directly.
10469	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10470	*
10471	* @param pVCpu The cross context virtual CPU structure.
10472	* @param pVmxTransient The VMX-transient structure.
10473	* @param fStepping Whether we are single-stepping the guest in the
10474	* hypervisor debugger. Makes us ignore some of the reasons
10475	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10476	* if event dispatching took place.
10477	*/
10478	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10479	{
10480	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10481
10482	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10483	if (pVmxTransient->fIsNestedGuest)
10484	{
10485	RT_NOREF2(pVCpu, fStepping);
10486	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10487	return VINF_EM_RESCHEDULE_REM;
10488	}
10489	#endif
10490
10491	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10492	PGMRZDynMapFlushAutoSet(pVCpu);
10493	#endif
10494
10495	/*
10496	* Check and process force flag actions, some of which might require us to go back to ring-3.
10497	*/
10498	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10499	if (rcStrict == VINF_SUCCESS)
10500	{ /* FFs don't get set all the time. */ }
10501	else
10502	return rcStrict;
10503
10504	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10505	/*
10506	* Switch to the nested-guest VMCS as we may have transitioned into executing
10507	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10508	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10509	*
10510	* We do this as late as possible to minimize (though not completely remove)
10511	* clearing/loading VMCS again due to premature trips to ring-3 above.
10512	*/
10513	if (pVmxTransient->fIsNestedGuest)
10514	{
10515	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10516	{
10517	/*
10518	* Ensure we have synced everything from the guest VMCS and also flag that
10519	* that we need to export the full (nested) guest-CPU context to the
10520	* nested-guest VMCS.
10521	*/
10522	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10523	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10524
10525	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10526	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10527	if (RT_LIKELY(rc == VINF_SUCCESS))
10528	{
10529	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10530	ASMSetFlags(fEFlags);
10531	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10532
10533	/*
10534	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10535	* flag that we need to update the host MSR values there. Even if we decide
10536	* in the future to share the VM-exit MSR-store area page with the guest,
10537	* if its content differs, we would have to update the host MSRs anyway.
10538	*/
10539	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10540	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10541	}
10542	else
10543	{
10544	ASMSetFlags(fEFlags);
10545	return rc;
10546	}
10547	}
10548
10549	/*
10550	* Merge guest VMCS controls with the nested-guest VMCS controls.
10551	*
10552	* Even if we have not executed the guest prior to this (e.g. when resuming
10553	* from a saved state), we should be okay with merging controls as we
10554	* initialize the guest VMCS controls as part of VM setup phase.
10555	*/
10556	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10557	{
10558	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10559	AssertRCReturn(rc, rc);
10560	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10561	}
10562	}
10563	#endif
10564
10565	/*
10566	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10567	*/
10568	PVM pVM = pVCpu->CTX_SUFF(pVM);
10569	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10570	&& hmR0VmxIsProcCtls2Set(pVCpu, pVmxTransient, VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10571	&& PDMHasApic(pVM))
10572	{
10573	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10574	AssertRCReturn(rc, rc);
10575	}
10576
10577	/*
10578	* Evaluate events to be injected into the guest.
10579	*
10580	* Events in TRPM can be injected without inspecting the guest state.
10581	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10582	* guest to cause a VM-exit the next time they are ready to receive the event.
10583	*/
10584	if (TRPMHasTrap(pVCpu))
10585	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10586
10587	uint32_t fIntrState;
10588	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10589
10590	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10591	/*
10592	* While evaluating pending events if something failed (unlikely) or if we were
10593	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10594	*/
10595	if ( rcStrict != VINF_SUCCESS
10596	\|\| ( pVmxTransient->fIsNestedGuest
10597	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10598	return rcStrict;
10599	#endif
10600
10601	/*
10602	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10603	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10604	* also result in triple-faulting the VM.
10605	*
10606	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10607	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10608	*/
10609	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10610	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10611	{ /* likely */ }
10612	else
10613	{
10614	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10615	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10616	return rcStrict;
10617	}
10618
10619	/*
10620	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10621	* import CR3 themselves. We will need to update them here, as even as late as the above
10622	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10623	* the below force flags to be set.
10624	*/
10625	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10626	{
10627	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10628	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10629	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10630	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10631	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10632	}
10633	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10634	{
10635	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10636	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10637	}
10638
10639	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10640	/* Paranoia. */
10641	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10642	#endif
10643
10644	/*
10645	* No longjmps to ring-3 from this point on!!!
10646	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10647	* This also disables flushing of the R0-logger instance (if any).
10648	*/
10649	VMMRZCallRing3Disable(pVCpu);
10650
10651	/*
10652	* Export the guest state bits.
10653	*
10654	* We cannot perform longjmps while loading the guest state because we do not preserve the
10655	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10656	* CPU migration.
10657	*
10658	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10659	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10660	*/
10661	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10662	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10663	{ /* likely */ }
10664	else
10665	{
10666	VMMRZCallRing3Enable(pVCpu);
10667	return rcStrict;
10668	}
10669
10670	/*
10671	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10672	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10673	* preemption disabled for a while. Since this is purely to aid the
10674	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10675	* disable interrupt on NT.
10676	*
10677	* We need to check for force-flags that could've possible been altered since we last
10678	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10679	* see @bugref{6398}).
10680	*
10681	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10682	* to ring-3 before executing guest code.
10683	*/
10684	pVmxTransient->fEFlags = ASMIntDisableFlags();
10685
10686	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10687	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10688	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10689	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10690	{
10691	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10692	{
10693	pVCpu->hm.s.Event.fPending = false;
10694
10695	/*
10696	* We've injected any pending events. This is really the point of no return (to ring-3).
10697	*
10698	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10699	* returns from this function, so don't enable them here.
10700	*/
10701	return VINF_SUCCESS;
10702	}
10703
10704	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10705	rcStrict = VINF_EM_RAW_INTERRUPT;
10706	}
10707	else
10708	{
10709	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10710	rcStrict = VINF_EM_RAW_TO_R3;
10711	}
10712
10713	ASMSetFlags(pVmxTransient->fEFlags);
10714	VMMRZCallRing3Enable(pVCpu);
10715
10716	return rcStrict;
10717	}
10718
10719
10720	/**
10721	* Final preparations before executing guest code using hardware-assisted VMX.
10722	*
10723	* We can no longer get preempted to a different host CPU and there are no returns
10724	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10725	* failures), this function is not intended to fail sans unrecoverable hardware
10726	* errors.
10727	*
10728	* @param pVCpu The cross context virtual CPU structure.
10729	* @param pVmxTransient The VMX-transient structure.
10730	*
10731	* @remarks Called with preemption disabled.
10732	* @remarks No-long-jump zone!!!
10733	*/
10734	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10735	{
10736	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10737	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10738	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10739	Assert(!pVCpu->hm.s.Event.fPending);
10740
10741	/*
10742	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10743	*/
10744	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10745	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10746
10747	PVM pVM = pVCpu->CTX_SUFF(pVM);
10748	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10749
10750	if (!CPUMIsGuestFPUStateActive(pVCpu))
10751	{
10752	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10753	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10754	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10755	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10756	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10757	}
10758
10759	/*
10760	* Re-save the host state bits as we may've been preempted (only happens when
10761	* thread-context hooks are used or when the VM start function changes).
10762	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10763	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10764	* see @bugref{8432}.
10765	*
10766	* This may also happen when switching to/from a nested-guest VMCS without leaving
10767	* ring-0.
10768	*/
10769	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10770	{
10771	int rc = hmR0VmxExportHostState(pVCpu);
10772	AssertRC(rc);
10773	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10774	}
10775	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10776
10777	/*
10778	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10779	*/
10780	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10781	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10782	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10783
10784	/*
10785	* Store status of the shared guest/host debug state at the time of VM-entry.
10786	*/
10787	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10788	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10789	{
10790	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10791	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10792	}
10793	else
10794	#endif
10795	{
10796	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10797	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10798	}
10799
10800	/*
10801	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10802	* more than one conditional check. The post-run side of our code shall determine
10803	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10804	*/
10805	if (pVmcsInfo->pbVirtApic)
10806	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10807
10808	/*
10809	* Update the host MSRs values in the VM-exit MSR-load area.
10810	*/
10811	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10812	{
10813	if (pVmcsInfo->cExitMsrLoad > 0)
10814	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10815	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10816	}
10817
10818	/*
10819	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10820	* VMX-preemption timer based on the next virtual sync clock deadline.
10821	*/
10822	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10823	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10824	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10825	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10826	{
10827	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10828	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10829	}
10830
10831	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10832	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10833	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10834	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10835
10836	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10837
10838	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10839	as we're about to start executing the guest . */
10840
10841	/*
10842	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10843	*
10844	* This is done this late as updating the TSC offsetting/preemption timer above
10845	* figures out if we can skip intercepting RDTSCP by calculating the number of
10846	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10847	*/
10848	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10849	{
10850	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10851	{
10852	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10853	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10854	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10855	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10856	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10857	AssertRC(rc);
10858	}
10859	else
10860	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10861	}
10862
10863	#ifdef VBOX_STRICT
10864	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10865	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10866	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10867	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10868	#endif
10869
10870	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10871	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10872	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10873	* see @bugref{9180#c54}. */
10874	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10875	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10876	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10877	#endif
10878	}
10879
10880
10881	/**
10882	* First C routine invoked after running guest code using hardware-assisted VMX.
10883	*
10884	* @param pVCpu The cross context virtual CPU structure.
10885	* @param pVmxTransient The VMX-transient structure.
10886	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10887	*
10888	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10889	*
10890	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10891	* unconditionally when it is safe to do so.
10892	*/
10893	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10894	{
10895	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10896
10897	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10898	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10899	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10900	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10901	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10902	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10903
10904	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10905	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10906	{
10907	uint64_t uGstTsc;
10908	if (!pVmxTransient->fIsNestedGuest)
10909	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10910	else
10911	{
10912	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10913	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10914	}
10915	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10916	}
10917
10918	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10919	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10920	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10921
10922	#if HC_ARCH_BITS == 64
10923	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10924	#endif
10925	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10926	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10927	and we need to leave it alone here. */
10928	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10929	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10930	#else
10931	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10932	#endif
10933	#ifdef VBOX_STRICT
10934	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10935	#endif
10936	Assert(!ASMIntAreEnabled());
10937	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10938	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10939
10940	/*
10941	* Save the basic VM-exit reason and check if the VM-entry failed.
10942	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10943	*/
10944	uint32_t uExitReason;
10945	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10946	AssertRC(rc);
10947	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10948	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10949
10950	/*
10951	* Check if VMLAUNCH/VMRESUME succeeded.
10952	* If this failed, we cause a guru meditation and cease further execution.
10953	*
10954	* However, if we are executing a nested-guest we might fail if we use the
10955	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
10956	*/
10957	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10958	{
10959	/*
10960	* Update the VM-exit history array here even if the VM-entry failed due to:
10961	* - Invalid guest state.
10962	* - MSR loading.
10963	* - Machine-check event.
10964	*
10965	* In any of the above cases we will still have a "valid" VM-exit reason
10966	* despite @a fVMEntryFailed being false.
10967	*
10968	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10969	*
10970	* Note! We don't have CS or RIP at this point. Will probably address that later
10971	* by amending the history entry added here.
10972	*/
10973	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10974	UINT64_MAX, uHostTsc);
10975
10976	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10977	{
10978	VMMRZCallRing3Enable(pVCpu);
10979
10980	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10981	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10982
10983	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10984	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10985	AssertRC(rc);
10986	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10987	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10988	AssertRC(rc);
10989	#else
10990	/*
10991	* Import the guest-interruptibility state always as we need it while evaluating
10992	* injecting events on re-entry.
10993	*
10994	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10995	* checking for real-mode while exporting the state because all bits that cause
10996	* mode changes wrt CR0 are intercepted.
10997	*/
10998	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10999	AssertRC(rc);
11000	#endif
11001
11002	/*
11003	* Sync the TPR shadow with our APIC state.
11004	*/
11005	if ( !pVmxTransient->fIsNestedGuest
11006	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11007	{
11008	Assert(pVmcsInfo->pbVirtApic);
11009	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11010	{
11011	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11012	AssertRC(rc);
11013	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11014	}
11015	}
11016
11017	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11018	return;
11019	}
11020	}
11021	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11022	else if (pVmxTransient->fIsNestedGuest)
11023	{
11024	# if 0
11025	/*
11026	* Copy the VM-instruction error field to the guest VMCS.
11027	*/
11028	/** @todo NSTVMX: Verify we're using the fast path. */
11029	uint32_t u32RoVmInstrError;
11030	rc = VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &u32RoVmInstrError);
11031	AssertRCReturn(rc, rc);
11032	PVMXVVMCS pGstVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
11033	pGstVmcs->u32RoVmInstrError = u32RoVmInstrError;
11034	/** @todo NSTVMX: Advance guest RIP and other fast path related restoration. */
11035	# else
11036	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11037	# endif
11038	}
11039	#endif
11040	else
11041	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11042
11043	VMMRZCallRing3Enable(pVCpu);
11044	}
11045
11046
11047	/**
11048	* Runs the guest code using hardware-assisted VMX the normal way.
11049	*
11050	* @returns VBox status code.
11051	* @param pVCpu The cross context virtual CPU structure.
11052	* @param pcLoops Pointer to the number of executed loops.
11053	*/
11054	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11055	{
11056	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11057	Assert(pcLoops);
11058	Assert(*pcLoops <= cMaxResumeLoops);
11059
11060	VMXTRANSIENT VmxTransient;
11061	RT_ZERO(VmxTransient);
11062	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11063
11064	/* Paranoia. */
11065	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11066	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11067
11068	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11069	for (;;)
11070	{
11071	Assert(!HMR0SuspendPending());
11072	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11073	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11074
11075	/*
11076	* Preparatory work for running nested-guest code, this may force us to
11077	* return to ring-3.
11078	*
11079	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11080	*/
11081	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11082	if (rcStrict != VINF_SUCCESS)
11083	break;
11084
11085	/* Interrupts are disabled at this point! */
11086	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11087	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11088	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11089	/* Interrupts are re-enabled at this point! */
11090
11091	/*
11092	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11093	*/
11094	if (RT_SUCCESS(rcRun))
11095	{ /* very likely */ }
11096	else
11097	{
11098	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11099	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11100	return rcRun;
11101	}
11102
11103	/*
11104	* Profile the VM-exit.
11105	*/
11106	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11107	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11108	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11109	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11110	HMVMX_START_EXIT_DISPATCH_PROF();
11111
11112	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11113
11114	/*
11115	* Handle the VM-exit.
11116	*/
11117	#ifdef HMVMX_USE_FUNCTION_TABLE
11118	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11119	#else
11120	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11121	#endif
11122	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11123	if (rcStrict == VINF_SUCCESS)
11124	{
11125	if (++(*pcLoops) <= cMaxResumeLoops)
11126	continue;
11127	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11128	rcStrict = VINF_EM_RAW_INTERRUPT;
11129	}
11130	break;
11131	}
11132
11133	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11134	return rcStrict;
11135	}
11136
11137	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11138	/**
11139	* Runs the nested-guest code using hardware-assisted VMX.
11140	*
11141	* @returns VBox status code.
11142	* @param pVCpu The cross context virtual CPU structure.
11143	* @param pcLoops Pointer to the number of executed loops.
11144	*
11145	* @sa hmR0VmxRunGuestCodeNormal.
11146	*/
11147	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11148	{
11149	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11150	Assert(pcLoops);
11151	Assert(*pcLoops <= cMaxResumeLoops);
11152
11153	VMXTRANSIENT VmxTransient;
11154	RT_ZERO(VmxTransient);
11155	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11156	VmxTransient.fIsNestedGuest = true;
11157
11158	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11159	for (;;)
11160	{
11161	Assert(!HMR0SuspendPending());
11162	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11163	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11164
11165	/*
11166	* Preparatory work for running guest code, this may force us to
11167	* return to ring-3.
11168	*
11169	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11170	*/
11171	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11172	if (rcStrict != VINF_SUCCESS)
11173	break;
11174
11175	/* Interrupts are disabled at this point! */
11176	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11177	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11178	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11179	/* Interrupts are re-enabled at this point! */
11180
11181	/*
11182	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11183	*/
11184	if (RT_SUCCESS(rcRun))
11185	{ /* very likely */ }
11186	else
11187	{
11188	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11189	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11190	return rcRun;
11191	}
11192
11193	/*
11194	* Profile the VM-exit.
11195	*/
11196	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11198	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11199	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11200	HMVMX_START_EXIT_DISPATCH_PROF();
11201
11202	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11203
11204	/*
11205	* Handle the VM-exit.
11206	*/
11207	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11208	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11209	if ( rcStrict == VINF_SUCCESS
11210	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11211	{
11212	if (++(*pcLoops) <= cMaxResumeLoops)
11213	continue;
11214	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11215	rcStrict = VINF_EM_RAW_INTERRUPT;
11216	}
11217	break;
11218	}
11219
11220	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11221	return rcStrict;
11222	}
11223	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11224
11225
11226	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11227	* probes.
11228	*
11229	* The following few functions and associated structure contains the bloat
11230	* necessary for providing detailed debug events and dtrace probes as well as
11231	* reliable host side single stepping. This works on the principle of
11232	* "subclassing" the normal execution loop and workers. We replace the loop
11233	* method completely and override selected helpers to add necessary adjustments
11234	* to their core operation.
11235	*
11236	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11237	* any performance for debug and analysis features.
11238	*
11239	* @{
11240	*/
11241
11242	/**
11243	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11244	* the debug run loop.
11245	*/
11246	typedef struct VMXRUNDBGSTATE
11247	{
11248	/** The RIP we started executing at. This is for detecting that we stepped. */
11249	uint64_t uRipStart;
11250	/** The CS we started executing with. */
11251	uint16_t uCsStart;
11252
11253	/** Whether we've actually modified the 1st execution control field. */
11254	bool fModifiedProcCtls : 1;
11255	/** Whether we've actually modified the 2nd execution control field. */
11256	bool fModifiedProcCtls2 : 1;
11257	/** Whether we've actually modified the exception bitmap. */
11258	bool fModifiedXcptBitmap : 1;
11259
11260	/** We desire the modified the CR0 mask to be cleared. */
11261	bool fClearCr0Mask : 1;
11262	/** We desire the modified the CR4 mask to be cleared. */
11263	bool fClearCr4Mask : 1;
11264	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11265	uint32_t fCpe1Extra;
11266	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11267	uint32_t fCpe1Unwanted;
11268	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11269	uint32_t fCpe2Extra;
11270	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11271	uint32_t bmXcptExtra;
11272	/** The sequence number of the Dtrace provider settings the state was
11273	* configured against. */
11274	uint32_t uDtraceSettingsSeqNo;
11275	/** VM-exits to check (one bit per VM-exit). */
11276	uint32_t bmExitsToCheck[3];
11277
11278	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11279	uint32_t fProcCtlsInitial;
11280	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11281	uint32_t fProcCtls2Initial;
11282	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11283	uint32_t bmXcptInitial;
11284	} VMXRUNDBGSTATE;
11285	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11286	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11287
11288
11289	/**
11290	* Initializes the VMXRUNDBGSTATE structure.
11291	*
11292	* @param pVCpu The cross context virtual CPU structure of the
11293	* calling EMT.
11294	* @param pVmxTransient The VMX-transient structure.
11295	* @param pDbgState The debug state to initialize.
11296	*/
11297	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11298	{
11299	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11300	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11301
11302	pDbgState->fModifiedProcCtls = false;
11303	pDbgState->fModifiedProcCtls2 = false;
11304	pDbgState->fModifiedXcptBitmap = false;
11305	pDbgState->fClearCr0Mask = false;
11306	pDbgState->fClearCr4Mask = false;
11307	pDbgState->fCpe1Extra = 0;
11308	pDbgState->fCpe1Unwanted = 0;
11309	pDbgState->fCpe2Extra = 0;
11310	pDbgState->bmXcptExtra = 0;
11311	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11312	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11313	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11314	}
11315
11316
11317	/**
11318	* Updates the VMSC fields with changes requested by @a pDbgState.
11319	*
11320	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11321	* immediately before executing guest code, i.e. when interrupts are disabled.
11322	* We don't check status codes here as we cannot easily assert or return in the
11323	* latter case.
11324	*
11325	* @param pVCpu The cross context virtual CPU structure.
11326	* @param pVmxTransient The VMX-transient structure.
11327	* @param pDbgState The debug state.
11328	*/
11329	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11330	{
11331	/*
11332	* Ensure desired flags in VMCS control fields are set.
11333	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11334	*
11335	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11336	* there should be no stale data in pCtx at this point.
11337	*/
11338	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11339	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11340	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11341	{
11342	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11343	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11344	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11345	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11346	pDbgState->fModifiedProcCtls = true;
11347	}
11348
11349	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11350	{
11351	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11352	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11353	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11354	pDbgState->fModifiedProcCtls2 = true;
11355	}
11356
11357	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11358	{
11359	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11360	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11361	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11362	pDbgState->fModifiedXcptBitmap = true;
11363	}
11364
11365	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11366	{
11367	pVmcsInfo->u64Cr0Mask = 0;
11368	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11369	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11370	}
11371
11372	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11373	{
11374	pVmcsInfo->u64Cr4Mask = 0;
11375	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11376	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11377	}
11378
11379	NOREF(pVCpu);
11380	}
11381
11382
11383	/**
11384	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11385	* re-entry next time around.
11386	*
11387	* @returns Strict VBox status code (i.e. informational status codes too).
11388	* @param pVCpu The cross context virtual CPU structure.
11389	* @param pVmxTransient The VMX-transient structure.
11390	* @param pDbgState The debug state.
11391	* @param rcStrict The return code from executing the guest using single
11392	* stepping.
11393	*/
11394	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11395	VBOXSTRICTRC rcStrict)
11396	{
11397	/*
11398	* Restore VM-exit control settings as we may not reenter this function the
11399	* next time around.
11400	*/
11401	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11402
11403	/* We reload the initial value, trigger what we can of recalculations the
11404	next time around. From the looks of things, that's all that's required atm. */
11405	if (pDbgState->fModifiedProcCtls)
11406	{
11407	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11408	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11409	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11410	AssertRCReturn(rc2, rc2);
11411	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11412	}
11413
11414	/* We're currently the only ones messing with this one, so just restore the
11415	cached value and reload the field. */
11416	if ( pDbgState->fModifiedProcCtls2
11417	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11418	{
11419	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11420	AssertRCReturn(rc2, rc2);
11421	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11422	}
11423
11424	/* If we've modified the exception bitmap, we restore it and trigger
11425	reloading and partial recalculation the next time around. */
11426	if (pDbgState->fModifiedXcptBitmap)
11427	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11428
11429	return rcStrict;
11430	}
11431
11432
11433	/**
11434	* Configures VM-exit controls for current DBGF and DTrace settings.
11435	*
11436	* This updates @a pDbgState and the VMCS execution control fields to reflect
11437	* the necessary VM-exits demanded by DBGF and DTrace.
11438	*
11439	* @param pVCpu The cross context virtual CPU structure.
11440	* @param pVmxTransient The VMX-transient structure. May update
11441	* fUpdatedTscOffsettingAndPreemptTimer.
11442	* @param pDbgState The debug state.
11443	*/
11444	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11445	{
11446	/*
11447	* Take down the dtrace serial number so we can spot changes.
11448	*/
11449	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11450	ASMCompilerBarrier();
11451
11452	/*
11453	* We'll rebuild most of the middle block of data members (holding the
11454	* current settings) as we go along here, so start by clearing it all.
11455	*/
11456	pDbgState->bmXcptExtra = 0;
11457	pDbgState->fCpe1Extra = 0;
11458	pDbgState->fCpe1Unwanted = 0;
11459	pDbgState->fCpe2Extra = 0;
11460	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11461	pDbgState->bmExitsToCheck[i] = 0;
11462
11463	/*
11464	* Software interrupts (INT XXh) - no idea how to trigger these...
11465	*/
11466	PVM pVM = pVCpu->CTX_SUFF(pVM);
11467	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11468	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11469	{
11470	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11471	}
11472
11473	/*
11474	* INT3 breakpoints - triggered by #BP exceptions.
11475	*/
11476	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11477	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11478
11479	/*
11480	* Exception bitmap and XCPT events+probes.
11481	*/
11482	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11483	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11484	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11485
11486	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11487	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11488	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11489	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11490	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11491	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11492	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11493	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11494	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11495	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11496	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11497	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11498	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11499	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11500	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11501	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11502	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11503	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11504
11505	if (pDbgState->bmXcptExtra)
11506	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11507
11508	/*
11509	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11510	*
11511	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11512	* So, when adding/changing/removing please don't forget to update it.
11513	*
11514	* Some of the macros are picking up local variables to save horizontal space,
11515	* (being able to see it in a table is the lesser evil here).
11516	*/
11517	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11518	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11519	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11520	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11521	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11522	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11523	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11524	} else do { } while (0)
11525	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11526	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11527	{ \
11528	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11529	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11530	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11531	} else do { } while (0)
11532	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11533	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11534	{ \
11535	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11536	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11537	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11538	} else do { } while (0)
11539	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11540	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11541	{ \
11542	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11543	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11544	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11545	} else do { } while (0)
11546
11547	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11548	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11549	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11550	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11551	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11552
11553	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11554	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11555	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11556	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11557	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11558	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11559	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11560	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11561	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11562	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11563	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11564	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11565	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11566	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11567	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11568	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11569	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11570	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11571	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11572	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11573	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11574	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11575	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11576	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11577	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11578	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11579	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11580	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11581	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11582	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11583	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11584	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11585	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11586	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11587	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11588	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11589
11590	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11591	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11592	{
11593	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11594	\| CPUMCTX_EXTRN_APIC_TPR);
11595	AssertRC(rc);
11596
11597	#if 0 /** @todo fix me */
11598	pDbgState->fClearCr0Mask = true;
11599	pDbgState->fClearCr4Mask = true;
11600	#endif
11601	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11602	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11603	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11604	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11605	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11606	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11607	require clearing here and in the loop if we start using it. */
11608	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11609	}
11610	else
11611	{
11612	if (pDbgState->fClearCr0Mask)
11613	{
11614	pDbgState->fClearCr0Mask = false;
11615	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11616	}
11617	if (pDbgState->fClearCr4Mask)
11618	{
11619	pDbgState->fClearCr4Mask = false;
11620	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11621	}
11622	}
11623	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11624	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11625
11626	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11627	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11628	{
11629	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11630	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11631	}
11632	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11633	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11634
11635	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11636	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11637	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11638	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11639	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11640	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11641	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11642	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11643	#if 0 /** @todo too slow, fix handler. */
11644	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11645	#endif
11646	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11647
11648	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11649	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11650	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11651	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11652	{
11653	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11654	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11655	}
11656	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11657	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11658	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11659	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11660
11661	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11662	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11663	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11664	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11665	{
11666	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11667	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11668	}
11669	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11670	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11671	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11672	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11673
11674	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11675	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11676	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11677	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11678	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11679	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11680	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11681	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11682	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11683	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11684	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11685	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11686	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11687	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11688	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11689	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11690	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11691	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11692	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11693	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11694	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11695	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11696
11697	#undef IS_EITHER_ENABLED
11698	#undef SET_ONLY_XBM_IF_EITHER_EN
11699	#undef SET_CPE1_XBM_IF_EITHER_EN
11700	#undef SET_CPEU_XBM_IF_EITHER_EN
11701	#undef SET_CPE2_XBM_IF_EITHER_EN
11702
11703	/*
11704	* Sanitize the control stuff.
11705	*/
11706	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11707	if (pDbgState->fCpe2Extra)
11708	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11709	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11710	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11711	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11712	{
11713	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11714	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11715	}
11716
11717	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11718	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11719	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11720	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11721	}
11722
11723
11724	/**
11725	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11726	* appropriate.
11727	*
11728	* The caller has checked the VM-exit against the
11729	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11730	* already, so we don't have to do that either.
11731	*
11732	* @returns Strict VBox status code (i.e. informational status codes too).
11733	* @param pVCpu The cross context virtual CPU structure.
11734	* @param pVmxTransient The VMX-transient structure.
11735	* @param uExitReason The VM-exit reason.
11736	*
11737	* @remarks The name of this function is displayed by dtrace, so keep it short
11738	* and to the point. No longer than 33 chars long, please.
11739	*/
11740	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11741	{
11742	/*
11743	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11744	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11745	*
11746	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11747	* does. Must add/change/remove both places. Same ordering, please.
11748	*
11749	* Added/removed events must also be reflected in the next section
11750	* where we dispatch dtrace events.
11751	*/
11752	bool fDtrace1 = false;
11753	bool fDtrace2 = false;
11754	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11755	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11756	uint32_t uEventArg = 0;
11757	#define SET_EXIT(a_EventSubName) \
11758	do { \
11759	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11760	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11761	} while (0)
11762	#define SET_BOTH(a_EventSubName) \
11763	do { \
11764	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11765	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11766	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11767	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11768	} while (0)
11769	switch (uExitReason)
11770	{
11771	case VMX_EXIT_MTF:
11772	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11773
11774	case VMX_EXIT_XCPT_OR_NMI:
11775	{
11776	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11777	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11778	{
11779	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11780	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11781	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11782	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11783	{
11784	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11785	{
11786	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11787	uEventArg = pVmxTransient->uExitIntErrorCode;
11788	}
11789	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11790	switch (enmEvent1)
11791	{
11792	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11793	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11794	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11795	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11796	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11797	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11798	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11799	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11800	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11801	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11802	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11803	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11804	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11805	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11806	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11807	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11808	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11809	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11810	default: break;
11811	}
11812	}
11813	else
11814	AssertFailed();
11815	break;
11816
11817	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11818	uEventArg = idxVector;
11819	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11820	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11821	break;
11822	}
11823	break;
11824	}
11825
11826	case VMX_EXIT_TRIPLE_FAULT:
11827	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11828	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11829	break;
11830	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11831	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11832	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11833	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11834	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11835
11836	/* Instruction specific VM-exits: */
11837	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11838	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11839	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11840	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11841	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11842	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11843	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11844	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11845	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11846	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11847	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11848	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11849	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11850	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11851	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11852	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11853	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11854	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11855	case VMX_EXIT_MOV_CRX:
11856	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11857	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11858	SET_BOTH(CRX_READ);
11859	else
11860	SET_BOTH(CRX_WRITE);
11861	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11862	break;
11863	case VMX_EXIT_MOV_DRX:
11864	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11865	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11866	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11867	SET_BOTH(DRX_READ);
11868	else
11869	SET_BOTH(DRX_WRITE);
11870	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11871	break;
11872	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11873	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11874	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11875	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11876	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11877	case VMX_EXIT_GDTR_IDTR_ACCESS:
11878	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11879	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11880	{
11881	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11882	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11883	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11884	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11885	}
11886	break;
11887
11888	case VMX_EXIT_LDTR_TR_ACCESS:
11889	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11890	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11891	{
11892	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11893	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11894	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11895	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11896	}
11897	break;
11898
11899	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11900	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11901	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11902	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11903	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11904	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11905	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11906	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11907	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11908	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11909	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11910
11911	/* Events that aren't relevant at this point. */
11912	case VMX_EXIT_EXT_INT:
11913	case VMX_EXIT_INT_WINDOW:
11914	case VMX_EXIT_NMI_WINDOW:
11915	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11916	case VMX_EXIT_PREEMPT_TIMER:
11917	case VMX_EXIT_IO_INSTR:
11918	break;
11919
11920	/* Errors and unexpected events. */
11921	case VMX_EXIT_INIT_SIGNAL:
11922	case VMX_EXIT_SIPI:
11923	case VMX_EXIT_IO_SMI:
11924	case VMX_EXIT_SMI:
11925	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11926	case VMX_EXIT_ERR_MSR_LOAD:
11927	case VMX_EXIT_ERR_MACHINE_CHECK:
11928	case VMX_EXIT_PML_FULL:
11929	case VMX_EXIT_VIRTUALIZED_EOI:
11930	break;
11931
11932	default:
11933	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11934	break;
11935	}
11936	#undef SET_BOTH
11937	#undef SET_EXIT
11938
11939	/*
11940	* Dtrace tracepoints go first. We do them here at once so we don't
11941	* have to copy the guest state saving and stuff a few dozen times.
11942	* Down side is that we've got to repeat the switch, though this time
11943	* we use enmEvent since the probes are a subset of what DBGF does.
11944	*/
11945	if (fDtrace1 \|\| fDtrace2)
11946	{
11947	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11948	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11949	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11950	switch (enmEvent1)
11951	{
11952	/** @todo consider which extra parameters would be helpful for each probe. */
11953	case DBGFEVENT_END: break;
11954	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11955	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11956	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11957	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11958	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11959	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11960	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11961	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11962	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11963	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11964	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11965	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11966	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11967	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11968	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11969	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11970	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11971	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11972	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11973	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11974	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11975	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11976	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11977	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11978	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11979	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11980	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11981	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11982	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11983	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11984	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11985	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11986	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11987	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11988	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11989	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11990	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11991	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11992	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11993	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11994	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11995	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11996	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11997	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11998	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11999	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12000	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12001	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12002	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12003	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12004	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12005	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12006	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12007	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12008	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12009	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12010	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12011	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12012	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12013	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12014	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12015	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12016	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12017	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12018	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12019	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12020	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12021	}
12022	switch (enmEvent2)
12023	{
12024	/** @todo consider which extra parameters would be helpful for each probe. */
12025	case DBGFEVENT_END: break;
12026	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12027	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12028	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12029	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12030	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12031	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12032	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12033	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12034	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12035	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12036	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12037	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12038	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12039	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12040	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12041	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12042	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12043	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12044	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12045	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12046	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12047	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12048	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12049	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12050	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12051	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12052	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12053	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12054	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12055	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12056	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12057	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12058	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12059	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12060	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12061	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12062	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12063	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12064	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12065	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12066	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12067	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12068	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12069	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12070	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12071	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12072	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12073	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12074	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12075	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12076	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12077	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12078	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12079	}
12080	}
12081
12082	/*
12083	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12084	* the DBGF call will do a full check).
12085	*
12086	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12087	* Note! If we have to events, we prioritize the first, i.e. the instruction
12088	* one, in order to avoid event nesting.
12089	*/
12090	PVM pVM = pVCpu->CTX_SUFF(pVM);
12091	if ( enmEvent1 != DBGFEVENT_END
12092	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12093	{
12094	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12095	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12096	if (rcStrict != VINF_SUCCESS)
12097	return rcStrict;
12098	}
12099	else if ( enmEvent2 != DBGFEVENT_END
12100	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12101	{
12102	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12103	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12104	if (rcStrict != VINF_SUCCESS)
12105	return rcStrict;
12106	}
12107
12108	return VINF_SUCCESS;
12109	}
12110
12111
12112	/**
12113	* Single-stepping VM-exit filtering.
12114	*
12115	* This is preprocessing the VM-exits and deciding whether we've gotten far
12116	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12117	* handling is performed.
12118	*
12119	* @returns Strict VBox status code (i.e. informational status codes too).
12120	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12121	* @param pVmxTransient The VMX-transient structure.
12122	* @param pDbgState The debug state.
12123	*/
12124	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12125	{
12126	/*
12127	* Expensive (saves context) generic dtrace VM-exit probe.
12128	*/
12129	uint32_t const uExitReason = pVmxTransient->uExitReason;
12130	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12131	{ /* more likely */ }
12132	else
12133	{
12134	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12135	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12136	AssertRC(rc);
12137	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12138	}
12139
12140	/*
12141	* Check for host NMI, just to get that out of the way.
12142	*/
12143	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12144	{ /* normally likely */ }
12145	else
12146	{
12147	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12148	AssertRCReturn(rc2, rc2);
12149	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12150	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12151	return hmR0VmxExitHostNmi(pVCpu);
12152	}
12153
12154	/*
12155	* Check for single stepping event if we're stepping.
12156	*/
12157	if (pVCpu->hm.s.fSingleInstruction)
12158	{
12159	switch (uExitReason)
12160	{
12161	case VMX_EXIT_MTF:
12162	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12163
12164	/* Various events: */
12165	case VMX_EXIT_XCPT_OR_NMI:
12166	case VMX_EXIT_EXT_INT:
12167	case VMX_EXIT_TRIPLE_FAULT:
12168	case VMX_EXIT_INT_WINDOW:
12169	case VMX_EXIT_NMI_WINDOW:
12170	case VMX_EXIT_TASK_SWITCH:
12171	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12172	case VMX_EXIT_APIC_ACCESS:
12173	case VMX_EXIT_EPT_VIOLATION:
12174	case VMX_EXIT_EPT_MISCONFIG:
12175	case VMX_EXIT_PREEMPT_TIMER:
12176
12177	/* Instruction specific VM-exits: */
12178	case VMX_EXIT_CPUID:
12179	case VMX_EXIT_GETSEC:
12180	case VMX_EXIT_HLT:
12181	case VMX_EXIT_INVD:
12182	case VMX_EXIT_INVLPG:
12183	case VMX_EXIT_RDPMC:
12184	case VMX_EXIT_RDTSC:
12185	case VMX_EXIT_RSM:
12186	case VMX_EXIT_VMCALL:
12187	case VMX_EXIT_VMCLEAR:
12188	case VMX_EXIT_VMLAUNCH:
12189	case VMX_EXIT_VMPTRLD:
12190	case VMX_EXIT_VMPTRST:
12191	case VMX_EXIT_VMREAD:
12192	case VMX_EXIT_VMRESUME:
12193	case VMX_EXIT_VMWRITE:
12194	case VMX_EXIT_VMXOFF:
12195	case VMX_EXIT_VMXON:
12196	case VMX_EXIT_MOV_CRX:
12197	case VMX_EXIT_MOV_DRX:
12198	case VMX_EXIT_IO_INSTR:
12199	case VMX_EXIT_RDMSR:
12200	case VMX_EXIT_WRMSR:
12201	case VMX_EXIT_MWAIT:
12202	case VMX_EXIT_MONITOR:
12203	case VMX_EXIT_PAUSE:
12204	case VMX_EXIT_GDTR_IDTR_ACCESS:
12205	case VMX_EXIT_LDTR_TR_ACCESS:
12206	case VMX_EXIT_INVEPT:
12207	case VMX_EXIT_RDTSCP:
12208	case VMX_EXIT_INVVPID:
12209	case VMX_EXIT_WBINVD:
12210	case VMX_EXIT_XSETBV:
12211	case VMX_EXIT_RDRAND:
12212	case VMX_EXIT_INVPCID:
12213	case VMX_EXIT_VMFUNC:
12214	case VMX_EXIT_RDSEED:
12215	case VMX_EXIT_XSAVES:
12216	case VMX_EXIT_XRSTORS:
12217	{
12218	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12219	AssertRCReturn(rc, rc);
12220	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12221	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12222	return VINF_EM_DBG_STEPPED;
12223	break;
12224	}
12225
12226	/* Errors and unexpected events: */
12227	case VMX_EXIT_INIT_SIGNAL:
12228	case VMX_EXIT_SIPI:
12229	case VMX_EXIT_IO_SMI:
12230	case VMX_EXIT_SMI:
12231	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12232	case VMX_EXIT_ERR_MSR_LOAD:
12233	case VMX_EXIT_ERR_MACHINE_CHECK:
12234	case VMX_EXIT_PML_FULL:
12235	case VMX_EXIT_VIRTUALIZED_EOI:
12236	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12237	break;
12238
12239	default:
12240	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12241	break;
12242	}
12243	}
12244
12245	/*
12246	* Check for debugger event breakpoints and dtrace probes.
12247	*/
12248	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12249	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12250	{
12251	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12252	if (rcStrict != VINF_SUCCESS)
12253	return rcStrict;
12254	}
12255
12256	/*
12257	* Normal processing.
12258	*/
12259	#ifdef HMVMX_USE_FUNCTION_TABLE
12260	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12261	#else
12262	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12263	#endif
12264	}
12265
12266
12267	/**
12268	* Single steps guest code using hardware-assisted VMX.
12269	*
12270	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12271	* but single-stepping through the hypervisor debugger.
12272	*
12273	* @returns Strict VBox status code (i.e. informational status codes too).
12274	* @param pVCpu The cross context virtual CPU structure.
12275	* @param pcLoops Pointer to the number of executed loops.
12276	*
12277	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12278	*/
12279	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12280	{
12281	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12282	Assert(pcLoops);
12283	Assert(*pcLoops <= cMaxResumeLoops);
12284
12285	VMXTRANSIENT VmxTransient;
12286	RT_ZERO(VmxTransient);
12287	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12288
12289	/* Set HMCPU indicators. */
12290	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12291	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12292	pVCpu->hm.s.fDebugWantRdTscExit = false;
12293	pVCpu->hm.s.fUsingDebugLoop = true;
12294
12295	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12296	VMXRUNDBGSTATE DbgState;
12297	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12298	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12299
12300	/*
12301	* The loop.
12302	*/
12303	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12304	for (;;)
12305	{
12306	Assert(!HMR0SuspendPending());
12307	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12308	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12309	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12310
12311	/* Set up VM-execution controls the next two can respond to. */
12312	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12313
12314	/*
12315	* Preparatory work for running guest code, this may force us to
12316	* return to ring-3.
12317	*
12318	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12319	*/
12320	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12321	if (rcStrict != VINF_SUCCESS)
12322	break;
12323
12324	/* Interrupts are disabled at this point! */
12325	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12326
12327	/* Override any obnoxious code in the above two calls. */
12328	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12329
12330	/*
12331	* Finally execute the guest.
12332	*/
12333	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12334
12335	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12336	/* Interrupts are re-enabled at this point! */
12337
12338	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12339	if (RT_SUCCESS(rcRun))
12340	{ /* very likely */ }
12341	else
12342	{
12343	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12344	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12345	return rcRun;
12346	}
12347
12348	/* Profile the VM-exit. */
12349	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12350	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12351	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12352	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12353	HMVMX_START_EXIT_DISPATCH_PROF();
12354
12355	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12356
12357	/*
12358	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12359	*/
12360	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12361	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12362	if (rcStrict != VINF_SUCCESS)
12363	break;
12364	if (++(*pcLoops) > cMaxResumeLoops)
12365	{
12366	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12367	rcStrict = VINF_EM_RAW_INTERRUPT;
12368	break;
12369	}
12370
12371	/*
12372	* Stepping: Did the RIP change, if so, consider it a single step.
12373	* Otherwise, make sure one of the TFs gets set.
12374	*/
12375	if (fStepping)
12376	{
12377	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12378	AssertRC(rc);
12379	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12380	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12381	{
12382	rcStrict = VINF_EM_DBG_STEPPED;
12383	break;
12384	}
12385	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12386	}
12387
12388	/*
12389	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12390	*/
12391	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12392	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12393	}
12394
12395	/*
12396	* Clear the X86_EFL_TF if necessary.
12397	*/
12398	if (pVCpu->hm.s.fClearTrapFlag)
12399	{
12400	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12401	AssertRC(rc);
12402	pVCpu->hm.s.fClearTrapFlag = false;
12403	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12404	}
12405	/** @todo there seems to be issues with the resume flag when the monitor trap
12406	* flag is pending without being used. Seen early in bios init when
12407	* accessing APIC page in protected mode. */
12408
12409	/*
12410	* Restore VM-exit control settings as we may not re-enter this function the
12411	* next time around.
12412	*/
12413	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12414
12415	/* Restore HMCPU indicators. */
12416	pVCpu->hm.s.fUsingDebugLoop = false;
12417	pVCpu->hm.s.fDebugWantRdTscExit = false;
12418	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12419
12420	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12421	return rcStrict;
12422	}
12423
12424
12425	/** @} */
12426
12427
12428	/**
12429	* Checks if any expensive dtrace probes are enabled and we should go to the
12430	* debug loop.
12431	*
12432	* @returns true if we should use debug loop, false if not.
12433	*/
12434	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12435	{
12436	/* It's probably faster to OR the raw 32-bit counter variables together.
12437	Since the variables are in an array and the probes are next to one
12438	another (more or less), we have good locality. So, better read
12439	eight-nine cache lines ever time and only have one conditional, than
12440	128+ conditionals, right? */
12441	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12442	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12443	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12444	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12445	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12446	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12447	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12448	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12449	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12450	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12451	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12452	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12453	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12454	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12455	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12456	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12457	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12458	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12459	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12460	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12461	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12462	) != 0
12463	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12464	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12465	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12466	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12467	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12468	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12469	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12470	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12471	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12472	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12473	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12474	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12475	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12476	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12477	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12478	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12479	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12480	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12481	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12482	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12483	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12484	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12485	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12486	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12487	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12488	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12489	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12490	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12491	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12492	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12493	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12494	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12495	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12496	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12497	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12498	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12499	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12500	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12501	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12502	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12503	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12504	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12505	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12506	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12507	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12508	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12509	) != 0
12510	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12511	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12512	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12513	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12514	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12515	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12516	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12517	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12518	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12519	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12520	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12521	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12522	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12523	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12524	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12525	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12526	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12527	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12528	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12529	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12530	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12531	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12532	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12533	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12534	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12535	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12536	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12537	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12538	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12539	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12540	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12541	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12542	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12543	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12544	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12545	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12546	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12547	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12548	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12549	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12550	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12551	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12552	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12553	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12554	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12555	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12556	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12557	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12558	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12559	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12560	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12561	) != 0;
12562	}
12563
12564
12565	/**
12566	* Runs the guest using hardware-assisted VMX.
12567	*
12568	* @returns Strict VBox status code (i.e. informational status codes too).
12569	* @param pVCpu The cross context virtual CPU structure.
12570	*/
12571	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12572	{
12573	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12574	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12575	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12576	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12577
12578	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12579
12580	VBOXSTRICTRC rcStrict;
12581	uint32_t cLoops = 0;
12582	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12583	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12584	#else
12585	bool const fInNestedGuestMode = false;
12586	#endif
12587	if (!fInNestedGuestMode)
12588	{
12589	if ( !pVCpu->hm.s.fUseDebugLoop
12590	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12591	&& !DBGFIsStepping(pVCpu)
12592	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12593	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12594	else
12595	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12596	}
12597	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12598	else
12599	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12600
12601	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12602	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12603	#endif
12604
12605	if (rcStrict == VERR_EM_INTERPRETER)
12606	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12607	else if (rcStrict == VINF_EM_RESET)
12608	rcStrict = VINF_EM_TRIPLE_FAULT;
12609
12610	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12611	if (RT_FAILURE(rc2))
12612	{
12613	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12614	rcStrict = rc2;
12615	}
12616	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12617	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12618	return rcStrict;
12619	}
12620
12621
12622	#ifndef HMVMX_USE_FUNCTION_TABLE
12623	/**
12624	* Handles a guest VM-exit from hardware-assisted VMX execution.
12625	*
12626	* @returns Strict VBox status code (i.e. informational status codes too).
12627	* @param pVCpu The cross context virtual CPU structure.
12628	* @param pVmxTransient The VMX-transient structure.
12629	*/
12630	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12631	{
12632	#ifdef DEBUG_ramshankar
12633	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12634	do { \
12635	if (a_fSave != 0) \
12636	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12637	VBOXSTRICTRC rcStrict = a_CallExpr; \
12638	if (a_fSave != 0) \
12639	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12640	return rcStrict; \
12641	} while (0)
12642	#else
12643	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12644	#endif
12645	uint32_t const uExitReason = pVmxTransient->uExitReason;
12646	switch (uExitReason)
12647	{
12648	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12649	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12650	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12651	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12652	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12653	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12654	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12655	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12656	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12657	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12658	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12659	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12660	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12661	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12662	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12663	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12664	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12665	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12666	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12667	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12668	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12669	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12670	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12671	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12672	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12673	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12674	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12675	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12676	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12677	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12678	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12679	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12680	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12681	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12682	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12683	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12684	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12685	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12686	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12687	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12688	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12689	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12690	#else
12691	case VMX_EXIT_VMCLEAR:
12692	case VMX_EXIT_VMLAUNCH:
12693	case VMX_EXIT_VMPTRLD:
12694	case VMX_EXIT_VMPTRST:
12695	case VMX_EXIT_VMREAD:
12696	case VMX_EXIT_VMRESUME:
12697	case VMX_EXIT_VMWRITE:
12698	case VMX_EXIT_VMXOFF:
12699	case VMX_EXIT_VMXON:
12700	case VMX_EXIT_INVVPID:
12701	case VMX_EXIT_INVEPT:
12702	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12703	#endif
12704
12705	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12706	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12707	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12708
12709	case VMX_EXIT_INIT_SIGNAL:
12710	case VMX_EXIT_SIPI:
12711	case VMX_EXIT_IO_SMI:
12712	case VMX_EXIT_SMI:
12713	case VMX_EXIT_ERR_MSR_LOAD:
12714	case VMX_EXIT_ERR_MACHINE_CHECK:
12715	case VMX_EXIT_PML_FULL:
12716	case VMX_EXIT_VIRTUALIZED_EOI:
12717	case VMX_EXIT_GDTR_IDTR_ACCESS:
12718	case VMX_EXIT_LDTR_TR_ACCESS:
12719	case VMX_EXIT_APIC_WRITE:
12720	case VMX_EXIT_RDRAND:
12721	case VMX_EXIT_RSM:
12722	case VMX_EXIT_VMFUNC:
12723	case VMX_EXIT_ENCLS:
12724	case VMX_EXIT_RDSEED:
12725	case VMX_EXIT_XSAVES:
12726	case VMX_EXIT_XRSTORS:
12727	case VMX_EXIT_UMWAIT:
12728	case VMX_EXIT_TPAUSE:
12729	default:
12730	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12731	}
12732	#undef VMEXIT_CALL_RET
12733	}
12734	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12735
12736
12737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12738	/**
12739	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12740	*
12741	* @returns Strict VBox status code (i.e. informational status codes too).
12742	* @param pVCpu The cross context virtual CPU structure.
12743	* @param pVmxTransient The VMX-transient structure.
12744	*/
12745	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12746	{
12747	uint32_t const uExitReason = pVmxTransient->uExitReason;
12748	switch (uExitReason)
12749	{
12750	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12751	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12752	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12753	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12754	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12755
12756	/*
12757	* We shouldn't direct host physical interrupts to the nested-guest.
12758	*/
12759	case VMX_EXIT_EXT_INT:
12760	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12761
12762	/*
12763	* Instructions that cause VM-exits unconditionally or the condition is
12764	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12765	* happens, it's guaranteed to be a nested-guest VM-exit).
12766	*
12767	* - Provides VM-exit instruction length ONLY.
12768	*/
12769	case VMX_EXIT_CPUID: /* Unconditional. */
12770	case VMX_EXIT_VMCALL:
12771	case VMX_EXIT_GETSEC:
12772	case VMX_EXIT_INVD:
12773	case VMX_EXIT_XSETBV:
12774	case VMX_EXIT_VMLAUNCH:
12775	case VMX_EXIT_VMRESUME:
12776	case VMX_EXIT_VMXOFF:
12777	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
12778	case VMX_EXIT_VMFUNC:
12779	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12780
12781	/*
12782	* Instructions that cause VM-exits unconditionally or the condition is
12783	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12784	* happens, it's guaranteed to be a nested-guest VM-exit).
12785	*
12786	* - Provides VM-exit instruction length.
12787	* - Provides VM-exit information.
12788	* - Optionally provides Exit qualification.
12789	*
12790	* Since Exit qualification is 0 for all VM-exits where it is not
12791	* applicable, reading and passing it to the guest should produce
12792	* defined behavior.
12793	*
12794	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12795	*/
12796	case VMX_EXIT_INVEPT: /* Unconditional. */
12797	case VMX_EXIT_INVVPID:
12798	case VMX_EXIT_VMCLEAR:
12799	case VMX_EXIT_VMPTRLD:
12800	case VMX_EXIT_VMPTRST:
12801	case VMX_EXIT_VMXON:
12802	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
12803	case VMX_EXIT_LDTR_TR_ACCESS:
12804	case VMX_EXIT_RDRAND:
12805	case VMX_EXIT_RDSEED:
12806	case VMX_EXIT_XSAVES:
12807	case VMX_EXIT_XRSTORS:
12808	case VMX_EXIT_UMWAIT:
12809	case VMX_EXIT_TPAUSE:
12810	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12811
12812	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12813	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12814	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12815	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12816	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12817	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12818	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12819	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12820	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12821	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12822	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12823	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12824	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12825	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12826	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12827	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12828	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12829	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12830	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12831
12832	case VMX_EXIT_PREEMPT_TIMER:
12833	{
12834	/** @todo NSTVMX: Preempt timer. */
12835	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12836	}
12837
12838	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12839	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12840
12841	case VMX_EXIT_VMREAD:
12842	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12843
12844	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12845	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12846
12847	case VMX_EXIT_INIT_SIGNAL:
12848	case VMX_EXIT_SIPI:
12849	case VMX_EXIT_IO_SMI:
12850	case VMX_EXIT_SMI:
12851	case VMX_EXIT_ERR_MSR_LOAD:
12852	case VMX_EXIT_ERR_MACHINE_CHECK:
12853	case VMX_EXIT_PML_FULL:
12854	case VMX_EXIT_RSM:
12855	default:
12856	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12857	}
12858	}
12859	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12860
12861
12862	#ifdef VBOX_STRICT
12863	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12864	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12865	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12866
12867	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12868	do { \
12869	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12870	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12871	} while (0)
12872
12873	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12874	do { \
12875	AssertPtr((a_pVCpu)); \
12876	AssertPtr((a_pVmxTransient)); \
12877	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12878	Assert((a_pVmxTransient)->pVmcsInfo); \
12879	Assert(ASMIntAreEnabled()); \
12880	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12881	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12882	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12883	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12884	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12885	HMVMX_ASSERT_PREEMPT_CPUID(); \
12886	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12887	} while (0)
12888
12889	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12890	do { \
12891	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
12892	Assert((a_pVmxTransient)->fIsNestedGuest); \
12893	} while (0)
12894
12895	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12896	do { \
12897	Log4Func(("\n")); \
12898	} while (0)
12899	#else
12900	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12901	do { \
12902	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12903	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12904	} while (0)
12905
12906	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12907	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
12908
12909	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12910	#endif
12911
12912
12913	/**
12914	* Advances the guest RIP by the specified number of bytes.
12915	*
12916	* @param pVCpu The cross context virtual CPU structure.
12917	* @param cbInstr Number of bytes to advance the RIP by.
12918	*
12919	* @remarks No-long-jump zone!!!
12920	*/
12921	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12922	{
12923	/* Advance the RIP. */
12924	pVCpu->cpum.GstCtx.rip += cbInstr;
12925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12926
12927	/* Update interrupt inhibition. */
12928	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12929	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12930	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12931	}
12932
12933
12934	/**
12935	* Advances the guest RIP after reading it from the VMCS.
12936	*
12937	* @returns VBox status code, no informational status codes.
12938	* @param pVCpu The cross context virtual CPU structure.
12939	* @param pVmxTransient The VMX-transient structure.
12940	*
12941	* @remarks No-long-jump zone!!!
12942	*/
12943	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12944	{
12945	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12946	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12947	AssertRCReturn(rc, rc);
12948
12949	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12950	return VINF_SUCCESS;
12951	}
12952
12953
12954	/**
12955	* Handle a condition that occurred while delivering an event through the guest
12956	* IDT.
12957	*
12958	* @returns Strict VBox status code (i.e. informational status codes too).
12959	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12960	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12961	* to continue execution of the guest which will delivery the \#DF.
12962	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12963	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12964	*
12965	* @param pVCpu The cross context virtual CPU structure.
12966	* @param pVmxTransient The VMX-transient structure.
12967	*
12968	* @remarks No-long-jump zone!!!
12969	*/
12970	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12971	{
12972	/* Read the IDT vectoring info. and VM-exit interruption info. */
12973	{
12974	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12975	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12976	AssertRCReturn(rc, rc);
12977	}
12978
12979	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12980	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12981	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12982	{
12983	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12984	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12985
12986	/*
12987	* If the event was a software interrupt (generated with INT n) or a software exception
12988	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12989	* can handle the VM-exit and continue guest execution which will re-execute the
12990	* instruction rather than re-injecting the exception, as that can cause premature
12991	* trips to ring-3 before injection and involve TRPM which currently has no way of
12992	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12993	* the problem).
12994	*/
12995	IEMXCPTRAISE enmRaise;
12996	IEMXCPTRAISEINFO fRaiseInfo;
12997	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12998	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12999	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13000	{
13001	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13002	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13003	}
13004	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13005	{
13006	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13007	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13008	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13009
13010	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13011	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13012	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13013
13014	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13015
13016	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13017	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13018	{
13019	pVmxTransient->fVectoringPF = true;
13020	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13021	}
13022	}
13023	else
13024	{
13025	/*
13026	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13027	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13028	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13029	*/
13030	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13031	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13032	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13033	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13034	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13035	}
13036
13037	/*
13038	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13039	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13040	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13041	* subsequent VM-entry would fail.
13042	*
13043	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
13044	*/
13045	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13046	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13047	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13048	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13049	&& CPUMIsGuestNmiBlocking(pVCpu))
13050	{
13051	CPUMSetGuestNmiBlocking(pVCpu, false);
13052	}
13053
13054	switch (enmRaise)
13055	{
13056	case IEMXCPTRAISE_CURRENT_XCPT:
13057	{
13058	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13059	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13060	Assert(rcStrict == VINF_SUCCESS);
13061	break;
13062	}
13063
13064	case IEMXCPTRAISE_PREV_EVENT:
13065	{
13066	uint32_t u32ErrCode;
13067	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13068	{
13069	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13070	AssertRCReturn(rc, rc);
13071	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13072	}
13073	else
13074	u32ErrCode = 0;
13075
13076	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13077	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13078	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13079	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13080
13081	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13082	pVCpu->hm.s.Event.u32ErrCode));
13083	Assert(rcStrict == VINF_SUCCESS);
13084	break;
13085	}
13086
13087	case IEMXCPTRAISE_REEXEC_INSTR:
13088	Assert(rcStrict == VINF_SUCCESS);
13089	break;
13090
13091	case IEMXCPTRAISE_DOUBLE_FAULT:
13092	{
13093	/*
13094	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13095	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13096	*/
13097	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13098	{
13099	pVmxTransient->fVectoringDoublePF = true;
13100	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13101	pVCpu->cpum.GstCtx.cr2));
13102	rcStrict = VINF_SUCCESS;
13103	}
13104	else
13105	{
13106	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13107	hmR0VmxSetPendingXcptDF(pVCpu);
13108	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13109	uIdtVector, uExitVector));
13110	rcStrict = VINF_HM_DOUBLE_FAULT;
13111	}
13112	break;
13113	}
13114
13115	case IEMXCPTRAISE_TRIPLE_FAULT:
13116	{
13117	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13118	rcStrict = VINF_EM_RESET;
13119	break;
13120	}
13121
13122	case IEMXCPTRAISE_CPU_HANG:
13123	{
13124	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13125	rcStrict = VERR_EM_GUEST_CPU_HANG;
13126	break;
13127	}
13128
13129	default:
13130	{
13131	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13132	rcStrict = VERR_VMX_IPE_2;
13133	break;
13134	}
13135	}
13136	}
13137	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13138	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13139	&& uExitVector != X86_XCPT_DF
13140	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13141	{
13142	/*
13143	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13144	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13145	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13146	*/
13147	CPUMSetGuestNmiBlocking(pVCpu, true);
13148	Log4Func(("Set NMI blocking. fValid=%RTbool uExitReason=%u\n", VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo),
13149	pVmxTransient->uExitReason));
13150	}
13151
13152	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13153	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13154	return rcStrict;
13155	}
13156
13157
13158	/** @name VM-exit handlers.
13159	* @{
13160	*/
13161	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13162	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13163	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13164
13165	/**
13166	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
13167	*/
13168	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13169	{
13170	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13171	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
13172	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
13173	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
13174	return VINF_SUCCESS;
13175	return VINF_EM_RAW_INTERRUPT;
13176	}
13177
13178
13179	/**
13180	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
13181	* VM-exit.
13182	*/
13183	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13184	{
13185	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13186	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
13187
13188	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13189	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13190	AssertRCReturn(rc, rc);
13191
13192	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13193	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
13194	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
13195	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
13196
13197	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
13198	{
13199	/*
13200	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
13201	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
13202	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
13203	*
13204	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
13205	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
13206	*/
13207	return hmR0VmxExitHostNmi(pVCpu);
13208	}
13209
13210	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13211	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
13212	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
13213	{ /* likely */ }
13214	else
13215	{
13216	if (rcStrict == VINF_HM_DOUBLE_FAULT)
13217	rcStrict = VINF_SUCCESS;
13218	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13219	return rcStrict;
13220	}
13221
13222	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13223	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13224	switch (uIntType)
13225	{
13226	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
13227	Assert(uVector == X86_XCPT_DB);
13228	RT_FALL_THRU();
13229	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
13230	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
13231	RT_FALL_THRU();
13232	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
13233	{
13234	/*
13235	* If there's any exception caused as a result of event injection, the resulting
13236	* secondary/final execption will be pending, we shall continue guest execution
13237	* after injecting the event. The page-fault case is complicated and we manually
13238	* handle any currently pending event in hmR0VmxExitXcptPF.
13239	*/
13240	if (!pVCpu->hm.s.Event.fPending)
13241	{ /* likely */ }
13242	else if (uVector != X86_XCPT_PF)
13243	{
13244	rcStrict = VINF_SUCCESS;
13245	break;
13246	}
13247
13248	switch (uVector)
13249	{
13250	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
13251	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
13252	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
13253	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
13254	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
13255	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
13256
13257	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13258	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13259	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
13260	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13261	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
13262	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13263	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
13264	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13265	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
13266	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13267	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
13268	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13269	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
13270	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
13271	default:
13272	{
13273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
13274	if (pVmcsInfo->RealMode.fRealOnV86Active)
13275	{
13276	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
13277	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
13278	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
13279
13280	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
13281	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13282	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13283	AssertRCReturn(rc, rc);
13284	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
13285	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
13286	0 /* GCPtrFaultAddress */);
13287	rcStrict = VINF_SUCCESS;
13288	}
13289	else
13290	{
13291	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
13292	pVCpu->hm.s.u32HMError = uVector;
13293	rcStrict = VERR_VMX_UNEXPECTED_EXCEPTION;
13294	}
13295	break;
13296	}
13297	}
13298	break;
13299	}
13300
13301	default:
13302	{
13303	pVCpu->hm.s.u32HMError = uExitIntInfo;
13304	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
13305	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
13306	break;
13307	}
13308	}
13309	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
13310	return rcStrict;
13311	}
13312
13313
13314	/**
13315	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
13316	*/
13317	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13318	{
13319	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13320
13321	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
13322	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13323	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
13324	AssertRCReturn(rc, rc);
13325
13326	/* Evaluate and deliver pending events and resume guest execution. */
13327	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
13328	return VINF_SUCCESS;
13329	}
13330
13331
13332	/**
13333	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
13334	*/
13335	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13336	{
13337	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13338
13339	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13340	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
13341	{
13342	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
13343	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13344	}
13345
13346	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
13347
13348	/*
13349	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
13350	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
13351	*/
13352	uint32_t fIntrState;
13353	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13354	AssertRCReturn(rc, rc);
13355	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
13356	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
13357	{
13358	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
13359	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13360
13361	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
13362	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
13363	AssertRCReturn(rc, rc);
13364	}
13365
13366	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
13367	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
13368	AssertRCReturn(rc, rc);
13369
13370	/* Evaluate and deliver pending events and resume guest execution. */
13371	return VINF_SUCCESS;
13372	}
13373
13374
13375	/**
13376	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
13377	*/
13378	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13379	{
13380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13381	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13382	}
13383
13384
13385	/**
13386	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13387	*/
13388	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13389	{
13390	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13391	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13392	}
13393
13394
13395	/**
13396	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13397	*/
13398	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13399	{
13400	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13401
13402	/*
13403	* Get the state we need and update the exit history entry.
13404	*/
13405	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13406	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13407	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13408	AssertRCReturn(rc, rc);
13409
13410	VBOXSTRICTRC rcStrict;
13411	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13412	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13413	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13414	if (!pExitRec)
13415	{
13416	/*
13417	* Regular CPUID instruction execution.
13418	*/
13419	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13420	if (rcStrict == VINF_SUCCESS)
13421	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13422	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13423	{
13424	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13425	rcStrict = VINF_SUCCESS;
13426	}
13427	}
13428	else
13429	{
13430	/*
13431	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13432	*/
13433	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13434	AssertRCReturn(rc2, rc2);
13435
13436	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13437	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13438
13439	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13440	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13441
13442	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13443	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13444	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13445	}
13446	return rcStrict;
13447	}
13448
13449
13450	/**
13451	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13452	*/
13453	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13454	{
13455	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13456
13457	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13458	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13459	AssertRCReturn(rc, rc);
13460
13461	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13462	return VINF_EM_RAW_EMULATE_INSTR;
13463
13464	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
13465	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13466	}
13467
13468
13469	/**
13470	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13471	*/
13472	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13473	{
13474	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13475
13476	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13477	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13478	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13479	AssertRCReturn(rc, rc);
13480
13481	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13482	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13483	{
13484	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13485	we must reset offsetting on VM-entry. See @bugref{6634}. */
13486	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13487	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13488	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13489	}
13490	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13491	{
13492	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13493	rcStrict = VINF_SUCCESS;
13494	}
13495	return rcStrict;
13496	}
13497
13498
13499	/**
13500	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13501	*/
13502	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13503	{
13504	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13505
13506	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13507	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13508	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13509	AssertRCReturn(rc, rc);
13510
13511	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13512	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13513	{
13514	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13515	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13516	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13517	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13518	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13519	}
13520	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13521	{
13522	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13523	rcStrict = VINF_SUCCESS;
13524	}
13525	return rcStrict;
13526	}
13527
13528
13529	/**
13530	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13531	*/
13532	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13533	{
13534	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13535
13536	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13537	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13538	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13539	AssertRCReturn(rc, rc);
13540
13541	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13542	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13543	if (RT_LIKELY(rc == VINF_SUCCESS))
13544	{
13545	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13546	Assert(pVmxTransient->cbInstr == 2);
13547	}
13548	else
13549	{
13550	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13551	rc = VERR_EM_INTERPRETER;
13552	}
13553	return rc;
13554	}
13555
13556
13557	/**
13558	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13559	*/
13560	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13561	{
13562	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13563
13564	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13565	if (EMAreHypercallInstructionsEnabled(pVCpu))
13566	{
13567	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13568	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13569	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13570	AssertRCReturn(rc, rc);
13571
13572	/* Perform the hypercall. */
13573	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13574	if (rcStrict == VINF_SUCCESS)
13575	{
13576	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13577	AssertRCReturn(rc, rc);
13578	}
13579	else
13580	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13581	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13582	\|\| RT_FAILURE(rcStrict));
13583
13584	/* If the hypercall changes anything other than guest's general-purpose registers,
13585	we would need to reload the guest changed bits here before VM-entry. */
13586	}
13587	else
13588	Log4Func(("Hypercalls not enabled\n"));
13589
13590	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13591	if (RT_FAILURE(rcStrict))
13592	{
13593	hmR0VmxSetPendingXcptUD(pVCpu);
13594	rcStrict = VINF_SUCCESS;
13595	}
13596
13597	return rcStrict;
13598	}
13599
13600
13601	/**
13602	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13603	*/
13604	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13605	{
13606	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13607	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13608
13609	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13610	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13611	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13612	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13613	AssertRCReturn(rc, rc);
13614
13615	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13616
13617	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13618	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13619	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13620	{
13621	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13622	rcStrict = VINF_SUCCESS;
13623	}
13624	else
13625	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
13626	VBOXSTRICTRC_VAL(rcStrict)));
13627	return rcStrict;
13628	}
13629
13630
13631	/**
13632	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13633	*/
13634	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13635	{
13636	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13637
13638	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13639	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
13640	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13641	AssertRCReturn(rc, rc);
13642
13643	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
13644	if (rcStrict == VINF_SUCCESS)
13645	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13646	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13647	{
13648	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13649	rcStrict = VINF_SUCCESS;
13650	}
13651
13652	return rcStrict;
13653	}
13654
13655
13656	/**
13657	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13658	*/
13659	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13660	{
13661	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13662
13663	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13664	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13665	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13666	AssertRCReturn(rc, rc);
13667
13668	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
13669	if (RT_SUCCESS(rcStrict))
13670	{
13671	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13672	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
13673	rcStrict = VINF_SUCCESS;
13674	}
13675
13676	return rcStrict;
13677	}
13678
13679
13680	/**
13681	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13682	* VM-exit.
13683	*/
13684	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13685	{
13686	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13687	return VINF_EM_RESET;
13688	}
13689
13690
13691	/**
13692	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13693	*/
13694	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13695	{
13696	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13697
13698	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13699	AssertRCReturn(rc, rc);
13700
13701	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13702	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13703	rc = VINF_SUCCESS;
13704	else
13705	rc = VINF_EM_HALT;
13706
13707	if (rc != VINF_SUCCESS)
13708	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13709	return rc;
13710	}
13711
13712
13713	/**
13714	* VM-exit handler for instructions that result in a \#UD exception delivered to
13715	* the guest.
13716	*/
13717	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13718	{
13719	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13720	hmR0VmxSetPendingXcptUD(pVCpu);
13721	return VINF_SUCCESS;
13722	}
13723
13724
13725	/**
13726	* VM-exit handler for expiry of the VMX-preemption timer.
13727	*/
13728	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13729	{
13730	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13731
13732	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13733	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13734
13735	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13736	PVM pVM = pVCpu->CTX_SUFF(pVM);
13737	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13738	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13739	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13740	}
13741
13742
13743	/**
13744	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13745	*/
13746	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13747	{
13748	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13749
13750	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13751	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13752	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13753	AssertRCReturn(rc, rc);
13754
13755	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13756	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13757	: HM_CHANGED_RAISED_XCPT_MASK);
13758
13759	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13760	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13761
13762	return rcStrict;
13763	}
13764
13765
13766	/**
13767	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13768	*/
13769	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13770	{
13771	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13772	/** @todo Use VM-exit instruction information. */
13773	return VERR_EM_INTERPRETER;
13774	}
13775
13776
13777	/**
13778	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
13779	* VM-exit.
13780	*/
13781	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13782	{
13783	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13784	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13785	AssertRCReturn(rc, rc);
13786
13787	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13788	if (RT_FAILURE(rc))
13789	return rc;
13790
13791	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13792	NOREF(uInvalidReason);
13793
13794	#ifdef VBOX_STRICT
13795	uint32_t fIntrState;
13796	RTHCUINTREG uHCReg;
13797	uint64_t u64Val;
13798	uint32_t u32Val;
13799	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13800	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13801	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13802	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13803	AssertRCReturn(rc, rc);
13804
13805	Log4(("uInvalidReason %u\n", uInvalidReason));
13806	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13807	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13808	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13809	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13810
13811	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13812	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13813	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13814	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13815	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13816	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13817	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13818	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13819	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13820	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13821	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13822	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13823
13824	hmR0DumpRegs(pVCpu);
13825	#endif
13826
13827	return VERR_VMX_INVALID_GUEST_STATE;
13828	}
13829
13830	/**
13831	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
13832	*/
13833	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13834	{
13835	/*
13836	* Cummulative notes of all recognized but unexpected VM-exits.
13837	*
13838	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
13839	* nested-paging is used.
13840	*
13841	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
13842	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
13843	* this function (and thereby stop VM execution) for handling such instructions.
13844	*
13845	*
13846	* VMX_EXIT_INIT_SIGNAL:
13847	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13848	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
13849	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
13850	*
13851	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
13852	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
13853	* See Intel spec. "23.8 Restrictions on VMX operation".
13854	*
13855	* VMX_EXIT_SIPI:
13856	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
13857	* activity state is used. We don't make use of it as our guests don't have direct
13858	* access to the host local APIC.
13859	*
13860	* See Intel spec. 25.3 "Other Causes of VM-exits".
13861	*
13862	* VMX_EXIT_IO_SMI:
13863	* VMX_EXIT_SMI:
13864	* This can only happen if we support dual-monitor treatment of SMI, which can be
13865	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
13866	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
13867	* VMX root mode or receive an SMI. If we get here, something funny is going on.
13868	*
13869	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13870	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13871	*
13872	* VMX_EXIT_ERR_MSR_LOAD:
13873	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
13874	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
13875	* execution.
13876	*
13877	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
13878	*
13879	* VMX_EXIT_ERR_MACHINE_CHECK:
13880	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
13881	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
13882	* #MC exception abort class exception is raised. We thus cannot assume a
13883	* reasonable chance of continuing any sort of execution and we bail.
13884	*
13885	* See Intel spec. 15.1 "Machine-check Architecture".
13886	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
13887	*
13888	* VMX_EXIT_PML_FULL:
13889	* VMX_EXIT_VIRTUALIZED_EOI:
13890	* VMX_EXIT_APIC_WRITE:
13891	* We do not currently support any of these features and thus they are all unexpected
13892	* VM-exits.
13893	*
13894	* VMX_EXIT_GDTR_IDTR_ACCESS:
13895	* VMX_EXIT_LDTR_TR_ACCESS:
13896	* VMX_EXIT_RDRAND:
13897	* VMX_EXIT_RSM:
13898	* VMX_EXIT_VMFUNC:
13899	* VMX_EXIT_ENCLS:
13900	* VMX_EXIT_RDSEED:
13901	* VMX_EXIT_XSAVES:
13902	* VMX_EXIT_XRSTORS:
13903	* VMX_EXIT_UMWAIT:
13904	* VMX_EXIT_TPAUSE:
13905	* These VM-exits are -not- caused unconditionally by execution of the corresponding
13906	* instruction. Any VM-exit for these instructions indicate a hardware problem,
13907	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
13908	*
13909	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
13910	*/
13911	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13912	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
13913	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
13914	}
13915
13916
13917	/**
13918	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13919	*/
13920	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13921	{
13922	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13923
13924	/** @todo Optimize this: We currently drag in in the whole MSR state
13925	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13926	* MSRs required. That would require changes to IEM and possibly CPUM too.
13927	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13928	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13929	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13930	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13931	switch (idMsr)
13932	{
13933	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13934	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13935	}
13936
13937	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13938	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13939	AssertRCReturn(rc, rc);
13940
13941	Log4Func(("ecx=%#RX32\n", idMsr));
13942
13943	#ifdef VBOX_STRICT
13944	if (hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
13945	{
13946	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13947	&& idMsr != MSR_K6_EFER)
13948	{
13949	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13950	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13951	}
13952	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13953	{
13954	Assert(pVmcsInfo->pvMsrBitmap);
13955	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13956	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13957	{
13958	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13959	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
13960	}
13961	}
13962	}
13963	#endif
13964
13965	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13967	if (rcStrict == VINF_SUCCESS)
13968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13969	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13970	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13971	{
13972	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13973	rcStrict = VINF_SUCCESS;
13974	}
13975	else
13976	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13977
13978	return rcStrict;
13979	}
13980
13981
13982	/**
13983	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13984	*/
13985	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13986	{
13987	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13988
13989	/** @todo Optimize this: We currently drag in in the whole MSR state
13990	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13991	* MSRs required. That would require changes to IEM and possibly CPUM too.
13992	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13993	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13994	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13995
13996	/*
13997	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13998	* Although we don't need to fetch the base as it will be overwritten shortly, while
13999	* loading guest-state we would also load the entire segment register including limit
14000	* and attributes and thus we need to load them here.
14001	*/
14002	switch (idMsr)
14003	{
14004	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
14005	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
14006	}
14007
14008	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14009	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14010	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
14011	AssertRCReturn(rc, rc);
14012
14013	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
14014
14015	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
14016	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
14017
14018	if (rcStrict == VINF_SUCCESS)
14019	{
14020	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14021
14022	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
14023	if ( idMsr == MSR_IA32_APICBASE
14024	\|\| ( idMsr >= MSR_IA32_X2APIC_START
14025	&& idMsr <= MSR_IA32_X2APIC_END))
14026	{
14027	/*
14028	* We've already saved the APIC related guest-state (TPR) in post-run phase.
14029	* When full APIC register virtualization is implemented we'll have to make
14030	* sure APIC state is saved from the VMCS before IEM changes it.
14031	*/
14032	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
14033	}
14034	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
14035	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14036	else if (idMsr == MSR_K6_EFER)
14037	{
14038	/*
14039	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
14040	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
14041	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
14042	*/
14043	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
14044	}
14045
14046	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
14047	if (!hmR0VmxIsProcCtlsSet(pVCpu, pVmxTransient, VMX_PROC_CTLS_USE_MSR_BITMAPS))
14048	{
14049	switch (idMsr)
14050	{
14051	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
14052	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
14053	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
14054	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
14055	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
14056	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
14057	default:
14058	{
14059	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14060	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
14061	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14062	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
14063	break;
14064	}
14065	}
14066	}
14067	#ifdef VBOX_STRICT
14068	else
14069	{
14070	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
14071	switch (idMsr)
14072	{
14073	case MSR_IA32_SYSENTER_CS:
14074	case MSR_IA32_SYSENTER_EIP:
14075	case MSR_IA32_SYSENTER_ESP:
14076	case MSR_K8_FS_BASE:
14077	case MSR_K8_GS_BASE:
14078	{
14079	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
14080	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14081	}
14082
14083	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
14084	default:
14085	{
14086	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
14087	{
14088	/* EFER MSR writes are always intercepted. */
14089	if (idMsr != MSR_K6_EFER)
14090	{
14091	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
14092	idMsr));
14093	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14094	}
14095	}
14096
14097	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
14098	{
14099	Assert(pVmcsInfo->pvMsrBitmap);
14100	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
14101	if (fMsrpm & VMXMSRPM_ALLOW_WR)
14102	{
14103	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
14104	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
14105	}
14106	}
14107	break;
14108	}
14109	}
14110	}
14111	#endif /* VBOX_STRICT */
14112	}
14113	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14114	{
14115	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14116	rcStrict = VINF_SUCCESS;
14117	}
14118	else
14119	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
14120
14121	return rcStrict;
14122	}
14123
14124
14125	/**
14126	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
14127	*/
14128	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14129	{
14130	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14131
14132	/** @todo The guest has likely hit a contended spinlock. We might want to
14133	* poke a schedule different guest VCPU. */
14134	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14135	if (RT_SUCCESS(rc))
14136	return VINF_EM_RAW_INTERRUPT;
14137
14138	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
14139	return rc;
14140	}
14141
14142
14143	/**
14144	* VM-exit handler for when the TPR value is lowered below the specified
14145	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
14146	*/
14147	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14148	{
14149	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14150	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
14151
14152	/*
14153	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
14154	* We'll re-evaluate pending interrupts and inject them before the next VM
14155	* entry so we can just continue execution here.
14156	*/
14157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
14158	return VINF_SUCCESS;
14159	}
14160
14161
14162	/**
14163	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
14164	* VM-exit.
14165	*
14166	* @retval VINF_SUCCESS when guest execution can continue.
14167	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
14168	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
14169	* incompatible guest state for VMX execution (real-on-v86 case).
14170	*/
14171	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14172	{
14173	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14174	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
14175
14176	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14177	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14178	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14179	AssertRCReturn(rc, rc);
14180
14181	VBOXSTRICTRC rcStrict;
14182	PVM pVM = pVCpu->CTX_SUFF(pVM);
14183	uint64_t const uExitQual = pVmxTransient->uExitQual;
14184	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
14185	switch (uAccessType)
14186	{
14187	/*
14188	* MOV to CRx.
14189	*/
14190	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
14191	{
14192	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14193	AssertRCReturn(rc, rc);
14194
14195	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
14196	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
14197	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14198	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14199
14200	/*
14201	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
14202	* - When nested paging isn't used.
14203	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
14204	* - We are executing in the VM debug loop.
14205	*/
14206	Assert( iCrReg != 3
14207	\|\| !pVM->hm.s.fNestedPaging
14208	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14209	\|\| pVCpu->hm.s.fUsingDebugLoop);
14210
14211	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
14212	Assert( iCrReg != 8
14213	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14214
14215	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14216	AssertMsg( rcStrict == VINF_SUCCESS
14217	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14218
14219	/*
14220	* This is a kludge for handling switches back to real mode when we try to use
14221	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
14222	* deal with special selector values, so we have to return to ring-3 and run
14223	* there till the selector values are V86 mode compatible.
14224	*
14225	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
14226	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
14227	* this function.
14228	*/
14229	if ( iCrReg == 0
14230	&& rcStrict == VINF_SUCCESS
14231	&& !pVM->hm.s.vmx.fUnrestrictedGuest
14232	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
14233	&& (uOldCr0 & X86_CR0_PE)
14234	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
14235	{
14236	/** @todo Check selectors rather than returning all the time. */
14237	Assert(!pVmxTransient->fIsNestedGuest);
14238	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
14239	rcStrict = VINF_EM_RESCHEDULE_REM;
14240	}
14241	break;
14242	}
14243
14244	/*
14245	* MOV from CRx.
14246	*/
14247	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
14248	{
14249	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
14250	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
14251
14252	/*
14253	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
14254	* - When nested paging isn't used.
14255	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
14256	* - We are executing in the VM debug loop.
14257	*/
14258	Assert( iCrReg != 3
14259	\|\| !pVM->hm.s.fNestedPaging
14260	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
14261	\|\| pVCpu->hm.s.fUsingDebugLoop);
14262
14263	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
14264	Assert( iCrReg != 8
14265	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
14266
14267	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
14268	break;
14269	}
14270
14271	/*
14272	* CLTS (Clear Task-Switch Flag in CR0).
14273	*/
14274	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
14275	{
14276	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
14277	break;
14278	}
14279
14280	/*
14281	* LMSW (Load Machine-Status Word into CR0).
14282	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
14283	*/
14284	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
14285	{
14286	RTGCPTR GCPtrEffDst;
14287	uint8_t const cbInstr = pVmxTransient->cbInstr;
14288	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
14289	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
14290	if (fMemOperand)
14291	{
14292	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14293	AssertRCReturn(rc, rc);
14294	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
14295	}
14296	else
14297	GCPtrEffDst = NIL_RTGCPTR;
14298	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
14299	break;
14300	}
14301
14302	default:
14303	{
14304	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
14305	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
14306	}
14307	}
14308
14309	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14310	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14311	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
14312
14313	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14314	NOREF(pVM);
14315	return rcStrict;
14316	}
14317
14318
14319	/**
14320	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14321	* VM-exit.
14322	*/
14323	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14324	{
14325	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14326	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14327
14328	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14329	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14330	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14331	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14332	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14333	\| CPUMCTX_EXTRN_EFER);
14334	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14335	AssertRCReturn(rc, rc);
14336
14337	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14338	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14339	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
14340	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14341	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14342	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14343	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14344	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
14345
14346	/*
14347	* Update exit history to see if this exit can be optimized.
14348	*/
14349	VBOXSTRICTRC rcStrict;
14350	PCEMEXITREC pExitRec = NULL;
14351	if ( !fGstStepping
14352	&& !fDbgStepping)
14353	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14354	!fIOString
14355	? !fIOWrite
14356	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14357	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14358	: !fIOWrite
14359	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14360	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14361	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14362	if (!pExitRec)
14363	{
14364	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
14365	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14366
14367	uint32_t const cbValue = s_aIOSizes[uIOSize];
14368	uint32_t const cbInstr = pVmxTransient->cbInstr;
14369	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14370	PVM pVM = pVCpu->CTX_SUFF(pVM);
14371	if (fIOString)
14372	{
14373	/*
14374	* INS/OUTS - I/O String instruction.
14375	*
14376	* Use instruction-information if available, otherwise fall back on
14377	* interpreting the instruction.
14378	*/
14379	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14380	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14381	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14382	if (fInsOutsInfo)
14383	{
14384	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14385	AssertRCReturn(rc2, rc2);
14386	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14387	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14388	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14389	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14390	if (fIOWrite)
14391	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14392	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14393	else
14394	{
14395	/*
14396	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14397	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14398	* See Intel Instruction spec. for "INS".
14399	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14400	*/
14401	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14402	}
14403	}
14404	else
14405	rcStrict = IEMExecOne(pVCpu);
14406
14407	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14408	fUpdateRipAlready = true;
14409	}
14410	else
14411	{
14412	/*
14413	* IN/OUT - I/O instruction.
14414	*/
14415	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14416	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
14417	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14418	if (fIOWrite)
14419	{
14420	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14421	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14422	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14423	&& !pCtx->eflags.Bits.u1TF)
14424	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14425	}
14426	else
14427	{
14428	uint32_t u32Result = 0;
14429	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14430	if (IOM_SUCCESS(rcStrict))
14431	{
14432	/* Save result of I/O IN instr. in AL/AX/EAX. */
14433	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14434	}
14435	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14436	&& !pCtx->eflags.Bits.u1TF)
14437	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14438	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14439	}
14440	}
14441
14442	if (IOM_SUCCESS(rcStrict))
14443	{
14444	if (!fUpdateRipAlready)
14445	{
14446	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14447	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14448	}
14449
14450	/*
14451	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14452	* while booting Fedora 17 64-bit guest.
14453	*
14454	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14455	*/
14456	if (fIOString)
14457	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14458
14459	/*
14460	* If any I/O breakpoints are armed, we need to check if one triggered
14461	* and take appropriate action.
14462	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14463	*/
14464	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14465	AssertRCReturn(rc, rc);
14466
14467	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14468	* execution engines about whether hyper BPs and such are pending. */
14469	uint32_t const uDr7 = pCtx->dr[7];
14470	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14471	&& X86_DR7_ANY_RW_IO(uDr7)
14472	&& (pCtx->cr4 & X86_CR4_DE))
14473	\|\| DBGFBpIsHwIoArmed(pVM)))
14474	{
14475	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14476
14477	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14478	VMMRZCallRing3Disable(pVCpu);
14479	HM_DISABLE_PREEMPT(pVCpu);
14480
14481	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14482
14483	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14484	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14485	{
14486	/* Raise #DB. */
14487	if (fIsGuestDbgActive)
14488	ASMSetDR6(pCtx->dr[6]);
14489	if (pCtx->dr[7] != uDr7)
14490	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14491
14492	hmR0VmxSetPendingXcptDB(pVCpu);
14493	}
14494	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14495	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14496	else if ( rcStrict2 != VINF_SUCCESS
14497	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14498	rcStrict = rcStrict2;
14499	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14500
14501	HM_RESTORE_PREEMPT();
14502	VMMRZCallRing3Enable(pVCpu);
14503	}
14504	}
14505
14506	#ifdef VBOX_STRICT
14507	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14508	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14509	Assert(!fIOWrite);
14510	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14511	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14512	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14513	Assert(fIOWrite);
14514	else
14515	{
14516	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14517	* statuses, that the VMM device and some others may return. See
14518	* IOM_SUCCESS() for guidance. */
14519	AssertMsg( RT_FAILURE(rcStrict)
14520	\|\| rcStrict == VINF_SUCCESS
14521	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14522	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14523	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14524	\|\| rcStrict == VINF_EM_RAW_TO_R3
14525	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14526	# endif
14527	}
14528	#endif
14529	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14530	}
14531	else
14532	{
14533	/*
14534	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14535	*/
14536	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14537	AssertRCReturn(rc2, rc2);
14538	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14539	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14540	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14541	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14542	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14543	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
14544
14545	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14546	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14547
14548	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14549	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14550	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14551	}
14552	return rcStrict;
14553	}
14554
14555
14556	/**
14557	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14558	* VM-exit.
14559	*/
14560	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14561	{
14562	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14563
14564	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14565	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14566	AssertRCReturn(rc, rc);
14567	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14568	{
14569	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14570	AssertRCReturn(rc, rc);
14571	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14572	{
14573	uint32_t uErrCode;
14574	RTGCUINTPTR GCPtrFaultAddress;
14575	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14576	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14577	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14578	if (fErrorCodeValid)
14579	{
14580	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14581	AssertRCReturn(rc, rc);
14582	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14583	}
14584	else
14585	uErrCode = 0;
14586
14587	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14588	&& uVector == X86_XCPT_PF)
14589	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14590	else
14591	GCPtrFaultAddress = 0;
14592
14593	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14594	AssertRCReturn(rc, rc);
14595
14596	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14597	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14598
14599	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14601	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14602	}
14603	}
14604
14605	/* Fall back to the interpreter to emulate the task-switch. */
14606	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14607	return VERR_EM_INTERPRETER;
14608	}
14609
14610
14611	/**
14612	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14613	*/
14614	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14615	{
14616	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14617
14618	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14619	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14620	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14621	AssertRCReturn(rc, rc);
14622	return VINF_EM_DBG_STEPPED;
14623	}
14624
14625
14626	/**
14627	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14628	*/
14629	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14630	{
14631	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14632	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14633
14634	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14635	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14636	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14637	{
14638	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14639	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14640	{
14641	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14642	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14643	}
14644	}
14645	else
14646	{
14647	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14648	rcStrict1 = VINF_SUCCESS;
14649	return rcStrict1;
14650	}
14651
14652	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14653	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14654	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14655	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14656	AssertRCReturn(rc, rc);
14657
14658	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14659	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14660	VBOXSTRICTRC rcStrict2;
14661	switch (uAccessType)
14662	{
14663	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14664	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14665	{
14666	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14667	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14668	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14669
14670	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14671	GCPhys &= PAGE_BASE_GC_MASK;
14672	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14673	PVM pVM = pVCpu->CTX_SUFF(pVM);
14674	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14675	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14676
14677	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14678	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14679	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14680	CPUMCTX2CORE(pCtx), GCPhys);
14681	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14682	if ( rcStrict2 == VINF_SUCCESS
14683	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14684	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14685	{
14686	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14687	\| HM_CHANGED_GUEST_APIC_TPR);
14688	rcStrict2 = VINF_SUCCESS;
14689	}
14690	break;
14691	}
14692
14693	default:
14694	Log4Func(("uAccessType=%#x\n", uAccessType));
14695	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14696	break;
14697	}
14698
14699	if (rcStrict2 != VINF_SUCCESS)
14700	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14701	return rcStrict2;
14702	}
14703
14704
14705	/**
14706	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14707	* VM-exit.
14708	*/
14709	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14710	{
14711	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14712
14713	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14714	if (pVmxTransient->fWasGuestDebugStateActive)
14715	{
14716	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14717	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14718	}
14719
14720	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14721	if ( !pVCpu->hm.s.fSingleInstruction
14722	&& !pVmxTransient->fWasHyperDebugStateActive)
14723	{
14724	Assert(!DBGFIsStepping(pVCpu));
14725	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14726
14727	/* Don't intercept MOV DRx any more. */
14728	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14729	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14730	AssertRCReturn(rc, rc);
14731
14732	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14733	VMMRZCallRing3Disable(pVCpu);
14734	HM_DISABLE_PREEMPT(pVCpu);
14735
14736	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14737	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14738	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14739
14740	HM_RESTORE_PREEMPT();
14741	VMMRZCallRing3Enable(pVCpu);
14742
14743	#ifdef VBOX_WITH_STATISTICS
14744	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14745	AssertRCReturn(rc, rc);
14746	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14748	else
14749	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14750	#endif
14751	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14752	return VINF_SUCCESS;
14753	}
14754
14755	/*
14756	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14757	* The EFER MSR is always up-to-date.
14758	* Update the segment registers and DR7 from the CPU.
14759	*/
14760	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14761	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14762	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14763	AssertRCReturn(rc, rc);
14764	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14765
14766	PVM pVM = pVCpu->CTX_SUFF(pVM);
14767	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14768	{
14769	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14770	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14771	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14772	if (RT_SUCCESS(rc))
14773	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14775	}
14776	else
14777	{
14778	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14779	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14780	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14782	}
14783
14784	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14785	if (RT_SUCCESS(rc))
14786	{
14787	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14788	AssertRCReturn(rc2, rc2);
14789	return VINF_SUCCESS;
14790	}
14791	return rc;
14792	}
14793
14794
14795	/**
14796	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14797	* Conditional VM-exit.
14798	*/
14799	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14800	{
14801	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14802	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14803
14804	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14805	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14806	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14807	{
14808	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14809	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14810	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14811	{
14812	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14813	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14814	}
14815	}
14816	else
14817	{
14818	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14819	rcStrict1 = VINF_SUCCESS;
14820	return rcStrict1;
14821	}
14822
14823	/*
14824	* Get sufficent state and update the exit history entry.
14825	*/
14826	RTGCPHYS GCPhys;
14827	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14828	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14829	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14830	AssertRCReturn(rc, rc);
14831
14832	VBOXSTRICTRC rcStrict;
14833	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14834	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14835	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14836	if (!pExitRec)
14837	{
14838	/*
14839	* If we succeed, resume guest execution.
14840	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14841	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14842	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14843	* weird case. See @bugref{6043}.
14844	*/
14845	PVM pVM = pVCpu->CTX_SUFF(pVM);
14846	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14847	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14848	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14849	if ( rcStrict == VINF_SUCCESS
14850	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14851	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14852	{
14853	/* Successfully handled MMIO operation. */
14854	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14855	\| HM_CHANGED_GUEST_APIC_TPR);
14856	rcStrict = VINF_SUCCESS;
14857	}
14858	}
14859	else
14860	{
14861	/*
14862	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14863	*/
14864	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14865	AssertRCReturn(rc2, rc2);
14866
14867	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14868	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14869
14870	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14871	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14872
14873	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14874	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14875	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14876	}
14877	return VBOXSTRICTRC_TODO(rcStrict);
14878	}
14879
14880
14881	/**
14882	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14883	* VM-exit.
14884	*/
14885	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14886	{
14887	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14888	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14889
14890	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14891	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14892	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14893	{
14894	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14895	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14896	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14897	}
14898	else
14899	{
14900	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14901	rcStrict1 = VINF_SUCCESS;
14902	return rcStrict1;
14903	}
14904
14905	RTGCPHYS GCPhys;
14906	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14907	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14908	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14909	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14910	AssertRCReturn(rc, rc);
14911
14912	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14913	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14914
14915	RTGCUINT uErrorCode = 0;
14916	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14917	uErrorCode \|= X86_TRAP_PF_ID;
14918	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14919	uErrorCode \|= X86_TRAP_PF_RW;
14920	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14921	uErrorCode \|= X86_TRAP_PF_P;
14922
14923	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14924
14925
14926	/* Handle the pagefault trap for the nested shadow table. */
14927	PVM pVM = pVCpu->CTX_SUFF(pVM);
14928	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14929
14930	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14931	pCtx->cs.Sel, pCtx->rip));
14932
14933	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14934	TRPMResetTrap(pVCpu);
14935
14936	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14937	if ( rcStrict2 == VINF_SUCCESS
14938	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14939	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14940	{
14941	/* Successfully synced our nested page tables. */
14942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14943	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14944	return VINF_SUCCESS;
14945	}
14946
14947	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14948	return rcStrict2;
14949	}
14950
14951	/** @} */
14952
14953	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14954	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14955	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14956
14957	/**
14958	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14959	*/
14960	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14961	{
14962	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14964
14965	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14966	AssertRCReturn(rc, rc);
14967
14968	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14969	{
14970	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14971	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14972
14973	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14974	* provides VM-exit instruction length. If this causes problem later,
14975	* disassemble the instruction like it's done on AMD-V. */
14976	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14977	AssertRCReturn(rc2, rc2);
14978	return rc;
14979	}
14980
14981	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14982	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14983	return rc;
14984	}
14985
14986
14987	/**
14988	* VM-exit exception handler for \#BP (Breakpoint exception).
14989	*/
14990	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14991	{
14992	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14993	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14994
14995	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14996	AssertRCReturn(rc, rc);
14997
14998	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14999	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15000	if (rc == VINF_EM_RAW_GUEST_TRAP)
15001	{
15002	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15003	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15004	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15005	AssertRCReturn(rc, rc);
15006
15007	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15008	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15009	}
15010
15011	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
15012	return rc;
15013	}
15014
15015
15016	/**
15017	* VM-exit exception handler for \#AC (alignment check exception).
15018	*/
15019	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15020	{
15021	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15022
15023	/*
15024	* Re-inject it. We'll detect any nesting before getting here.
15025	*/
15026	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15027	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15028	AssertRCReturn(rc, rc);
15029	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15030
15031	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15032	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15033	return VINF_SUCCESS;
15034	}
15035
15036
15037	/**
15038	* VM-exit exception handler for \#DB (Debug exception).
15039	*/
15040	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15041	{
15042	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
15044
15045	/*
15046	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
15047	*/
15048	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15049
15050	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
15051	uint64_t const uDR6 = X86_DR6_INIT_VAL
15052	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
15053	\| X86_DR6_BD \| X86_DR6_BS));
15054
15055	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15056	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
15057	Log6Func(("rc=%Rrc\n", rc));
15058	if (rc == VINF_EM_RAW_GUEST_TRAP)
15059	{
15060	/*
15061	* The exception was for the guest. Update DR6, DR7.GD and
15062	* IA32_DEBUGCTL.LBR before forwarding it.
15063	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
15064	*/
15065	VMMRZCallRing3Disable(pVCpu);
15066	HM_DISABLE_PREEMPT(pVCpu);
15067
15068	pCtx->dr[6] &= ~X86_DR6_B_MASK;
15069	pCtx->dr[6] \|= uDR6;
15070	if (CPUMIsGuestDebugStateActive(pVCpu))
15071	ASMSetDR6(pCtx->dr[6]);
15072
15073	HM_RESTORE_PREEMPT();
15074	VMMRZCallRing3Enable(pVCpu);
15075
15076	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
15077	AssertRCReturn(rc, rc);
15078
15079	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
15080	pCtx->dr[7] &= ~X86_DR7_GD;
15081
15082	/* Paranoia. */
15083	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
15084	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
15085
15086	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
15087	AssertRCReturn(rc, rc);
15088
15089	/*
15090	* Raise #DB in the guest.
15091	*
15092	* It is important to reflect exactly what the VM-exit gave us (preserving the
15093	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
15094	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
15095	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
15096	*
15097	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
15098	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
15099	*/
15100	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15101	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15102	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15103	AssertRCReturn(rc, rc);
15104	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15105	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15106	return VINF_SUCCESS;
15107	}
15108
15109	/*
15110	* Not a guest trap, must be a hypervisor related debug event then.
15111	* Update DR6 in case someone is interested in it.
15112	*/
15113	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
15114	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
15115	CPUMSetHyperDR6(pVCpu, uDR6);
15116
15117	return rc;
15118	}
15119
15120
15121	/**
15122	* Hacks its way around the lovely mesa driver's backdoor accesses.
15123	*
15124	* @sa hmR0SvmHandleMesaDrvGp.
15125	*/
15126	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15127	{
15128	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
15129	RT_NOREF(pCtx);
15130
15131	/* For now we'll just skip the instruction. */
15132	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15133	}
15134
15135
15136	/**
15137	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
15138	* backdoor logging w/o checking what it is running inside.
15139	*
15140	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
15141	* backdoor port and magic numbers loaded in registers.
15142	*
15143	* @returns true if it is, false if it isn't.
15144	* @sa hmR0SvmIsMesaDrvGp.
15145	*/
15146	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
15147	{
15148	/* 0xed: IN eAX,dx */
15149	uint8_t abInstr[1];
15150	if (pVmxTransient->cbInstr != sizeof(abInstr))
15151	return false;
15152
15153	/* Check that it is #GP(0). */
15154	if (pVmxTransient->uExitIntErrorCode != 0)
15155	return false;
15156
15157	/* Check magic and port. */
15158	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
15159	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
15160	if (pCtx->rax != UINT32_C(0x564d5868))
15161	return false;
15162	if (pCtx->dx != UINT32_C(0x5658))
15163	return false;
15164
15165	/* Flat ring-3 CS. */
15166	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
15167	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
15168	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
15169	if (pCtx->cs.Attr.n.u2Dpl != 3)
15170	return false;
15171	if (pCtx->cs.u64Base != 0)
15172	return false;
15173
15174	/* Check opcode. */
15175	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
15176	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
15177	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
15178	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
15179	if (RT_FAILURE(rc))
15180	return false;
15181	if (abInstr[0] != 0xed)
15182	return false;
15183
15184	return true;
15185	}
15186
15187
15188	/**
15189	* VM-exit exception handler for \#GP (General-protection exception).
15190	*
15191	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
15192	*/
15193	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15194	{
15195	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
15197
15198	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15199	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15200	if (pVmcsInfo->RealMode.fRealOnV86Active)
15201	{ /* likely */ }
15202	else
15203	{
15204	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15205	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
15206	#endif
15207	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
15208	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15209	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15210	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15211	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15212	AssertRCReturn(rc, rc);
15213	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
15214	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
15215
15216	if ( pVmxTransient->fIsNestedGuest
15217	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
15218	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
15219	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
15220	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15221	else
15222	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
15223	return rc;
15224	}
15225
15226	Assert(CPUMIsGuestInRealModeEx(pCtx));
15227	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
15228	Assert(!pVmxTransient->fIsNestedGuest);
15229
15230	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15231	AssertRCReturn(rc, rc);
15232
15233	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
15234	if (rcStrict == VINF_SUCCESS)
15235	{
15236	if (!CPUMIsGuestInRealModeEx(pCtx))
15237	{
15238	/*
15239	* The guest is no longer in real-mode, check if we can continue executing the
15240	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
15241	*/
15242	pVmcsInfo->RealMode.fRealOnV86Active = false;
15243	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
15244	{
15245	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
15246	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15247	}
15248	else
15249	{
15250	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
15251	rcStrict = VINF_EM_RESCHEDULE;
15252	}
15253	}
15254	else
15255	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15256	}
15257	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15258	{
15259	rcStrict = VINF_SUCCESS;
15260	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15261	}
15262	return VBOXSTRICTRC_VAL(rcStrict);
15263	}
15264
15265
15266	/**
15267	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15268	* the exception reported in the VMX transient structure back into the VM.
15269	*
15270	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15271	* up-to-date.
15272	*/
15273	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15274	{
15275	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15276	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15277	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15278	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
15279	("uVector=%#x u32XcptBitmap=%#X32\n",
15280	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15281	NOREF(pVmcsInfo);
15282	#endif
15283
15284	/*
15285	* Re-inject the exception into the guest. This cannot be a double-fault condition which
15286	* would have been handled while checking exits due to event delivery.
15287	*/
15288	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15289	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15290	AssertRCReturn(rc, rc);
15291	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15292
15293	#ifdef DEBUG_ramshankar
15294	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15295	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15296	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15297	#endif
15298
15299	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15300	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15301	return VINF_SUCCESS;
15302	}
15303
15304
15305	/**
15306	* VM-exit exception handler for \#PF (Page-fault exception).
15307	*/
15308	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15309	{
15310	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15311	PVM pVM = pVCpu->CTX_SUFF(pVM);
15312	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15313	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15314	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15315	AssertRCReturn(rc, rc);
15316
15317	if (!pVM->hm.s.fNestedPaging)
15318	{ /* likely */ }
15319	else
15320	{
15321	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15322	Assert(pVCpu->hm.s.fUsingDebugLoop);
15323	#endif
15324	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15325	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15326	{
15327	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15328	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15329	}
15330	else
15331	{
15332	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15333	hmR0VmxSetPendingXcptDF(pVCpu);
15334	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15335	}
15336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15337	return rc;
15338	}
15339
15340	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15341	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15342	if (pVmxTransient->fVectoringPF)
15343	{
15344	Assert(pVCpu->hm.s.Event.fPending);
15345	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15346	}
15347
15348	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15349	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15350	AssertRCReturn(rc, rc);
15351
15352	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15353	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15354
15355	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15356	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15357
15358	Log4Func(("#PF: rc=%Rrc\n", rc));
15359	if (rc == VINF_SUCCESS)
15360	{
15361	/*
15362	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15363	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15364	*/
15365	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15366	TRPMResetTrap(pVCpu);
15367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15368	return rc;
15369	}
15370
15371	if (rc == VINF_EM_RAW_GUEST_TRAP)
15372	{
15373	if (!pVmxTransient->fVectoringDoublePF)
15374	{
15375	/* It's a guest page fault and needs to be reflected to the guest. */
15376	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15377	TRPMResetTrap(pVCpu);
15378	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15379	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15380	uGstErrorCode, pVmxTransient->uExitQual);
15381	}
15382	else
15383	{
15384	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15385	TRPMResetTrap(pVCpu);
15386	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15387	hmR0VmxSetPendingXcptDF(pVCpu);
15388	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15389	}
15390
15391	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15392	return VINF_SUCCESS;
15393	}
15394
15395	TRPMResetTrap(pVCpu);
15396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15397	return rc;
15398	}
15399
15400
15401	/**
15402	* VM-exit helper for LMSW.
15403	*/
15404	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw,
15405	RTGCPTR GCPtrEffDst)
15406	{
15407	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15408	AssertRCReturn(rc, rc);
15409
15410	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
15411	AssertMsg( rcStrict == VINF_SUCCESS
15412	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15413
15414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15415	if (rcStrict == VINF_IEM_RAISED_XCPT)
15416	{
15417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15418	rcStrict = VINF_SUCCESS;
15419	}
15420
15421	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
15422	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15423	return rcStrict;
15424	}
15425
15426
15427	/**
15428	* VM-exit helper for CLTS.
15429	*/
15430	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
15431	{
15432	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15433	AssertRCReturn(rc, rc);
15434
15435	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
15436	AssertMsg( rcStrict == VINF_SUCCESS
15437	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15438
15439	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15440	if (rcStrict == VINF_IEM_RAISED_XCPT)
15441	{
15442	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15443	rcStrict = VINF_SUCCESS;
15444	}
15445
15446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
15447	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15448	return rcStrict;
15449	}
15450
15451
15452	/**
15453	* VM-exit helper for MOV from CRx (CRx read).
15454	*/
15455	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15456	{
15457	Assert(iCrReg < 16);
15458	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
15459
15460	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15461	AssertRCReturn(rc, rc);
15462
15463	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
15464	AssertMsg( rcStrict == VINF_SUCCESS
15465	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15466
15467	if (iGReg == X86_GREG_xSP)
15468	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
15469	else
15470	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15471	#ifdef VBOX_WITH_STATISTICS
15472	switch (iCrReg)
15473	{
15474	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
15475	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
15476	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
15477	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
15478	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
15479	}
15480	#endif
15481	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
15482	return rcStrict;
15483	}
15484
15485
15486	/**
15487	* VM-exit helper for MOV to CRx (CRx write).
15488	*/
15489	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15490	{
15491	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15492	AssertRCReturn(rc, rc);
15493
15494	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
15495	AssertMsg( rcStrict == VINF_SUCCESS
15496	\|\| rcStrict == VINF_IEM_RAISED_XCPT
15497	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15498
15499	switch (iCrReg)
15500	{
15501	case 0:
15502	{
15503	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
15504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
15505	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
15506	break;
15507	}
15508
15509	case 2:
15510	{
15511	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
15512	/* Nothing to do here, CR2 it's not part of the VMCS. */
15513	break;
15514	}
15515
15516	case 3:
15517	{
15518	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
15519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
15520	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
15521	break;
15522	}
15523
15524	case 4:
15525	{
15526	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
15527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
15528	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
15529	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
15530	break;
15531	}
15532
15533	case 8:
15534	{
15535	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
15536	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
15537	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
15538	break;
15539	}
15540
15541	default:
15542	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
15543	break;
15544	}
15545
15546	if (rcStrict == VINF_IEM_RAISED_XCPT)
15547	{
15548	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15549	rcStrict = VINF_SUCCESS;
15550	}
15551	return rcStrict;
15552	}
15553
15554
15555	/**
15556	* VM-exit helper for handling host NMIs.
15557	*/
15558	static VBOXSTRICTRC hmR0VmxExitHostNmi(PVMCPU pVCpu)
15559	{
15560	VMXDispatchHostNmi();
15561
15562	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
15563	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
15564	return VINF_SUCCESS;
15565	}
15566
15567
15568	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15569	/** @name VMX instruction handlers.
15570	* @{
15571	*/
15572	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15573	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15574	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15575
15576	/**
15577	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15578	*/
15579	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15580	{
15581	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15582
15583	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15584	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15585	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15586	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15587	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15588	AssertRCReturn(rc, rc);
15589
15590	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15591
15592	VMXVEXITINFO ExitInfo;
15593	RT_ZERO(ExitInfo);
15594	ExitInfo.uReason = pVmxTransient->uExitReason;
15595	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15596	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15597	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15598	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15599
15600	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15601	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15602	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15603	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15604	{
15605	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15606	rcStrict = VINF_SUCCESS;
15607	}
15608	return rcStrict;
15609	}
15610
15611
15612	/**
15613	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15614	*/
15615	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15616	{
15617	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15618
15619	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15620	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15621	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15622	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15623	AssertRCReturn(rc, rc);
15624
15625	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15626
15627	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15628	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15629	{
15630	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15631	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15632	}
15633	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15634	return rcStrict;
15635	}
15636
15637
15638	/**
15639	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15640	*/
15641	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15642	{
15643	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15644
15645	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15646	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15647	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15648	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15649	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15650	AssertRCReturn(rc, rc);
15651
15652	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15653
15654	VMXVEXITINFO ExitInfo;
15655	RT_ZERO(ExitInfo);
15656	ExitInfo.uReason = pVmxTransient->uExitReason;
15657	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15658	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15659	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15660	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15661
15662	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15663	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15664	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15665	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15666	{
15667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15668	rcStrict = VINF_SUCCESS;
15669	}
15670	return rcStrict;
15671	}
15672
15673
15674	/**
15675	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15676	*/
15677	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15678	{
15679	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15680
15681	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15682	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15683	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15684	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15685	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15686	AssertRCReturn(rc, rc);
15687
15688	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15689
15690	VMXVEXITINFO ExitInfo;
15691	RT_ZERO(ExitInfo);
15692	ExitInfo.uReason = pVmxTransient->uExitReason;
15693	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15694	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15695	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15696	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15697
15698	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15699	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15700	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15701	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15702	{
15703	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15704	rcStrict = VINF_SUCCESS;
15705	}
15706	return rcStrict;
15707	}
15708
15709
15710	/**
15711	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
15712	*/
15713	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15714	{
15715	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15716
15717	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15718	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15719	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15720	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15721	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15722	AssertRCReturn(rc, rc);
15723
15724	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15725
15726	VMXVEXITINFO ExitInfo;
15727	RT_ZERO(ExitInfo);
15728	ExitInfo.uReason = pVmxTransient->uExitReason;
15729	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15730	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15731	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15732	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15733	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15734
15735	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15736	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15737	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15738	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15739	{
15740	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15741	rcStrict = VINF_SUCCESS;
15742	}
15743	return rcStrict;
15744	}
15745
15746
15747	/**
15748	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15749	*/
15750	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15751	{
15752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15753
15754	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15755	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15756	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15757	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15758	AssertRCReturn(rc, rc);
15759
15760	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15761
15762	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15763	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15764	{
15765	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15766	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15767	}
15768	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15769	return rcStrict;
15770	}
15771
15772
15773	/**
15774	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
15775	*/
15776	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15777	{
15778	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15779
15780	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15781	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15782	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15783	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15784	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15785	AssertRCReturn(rc, rc);
15786
15787	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15788
15789	VMXVEXITINFO ExitInfo;
15790	RT_ZERO(ExitInfo);
15791	ExitInfo.uReason = pVmxTransient->uExitReason;
15792	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15793	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15794	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15795	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15796	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15797
15798	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15799	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15800	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15801	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15802	{
15803	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15804	rcStrict = VINF_SUCCESS;
15805	}
15806	return rcStrict;
15807	}
15808
15809
15810	/**
15811	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15812	*/
15813	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15814	{
15815	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15816
15817	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15818	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15819	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15820	AssertRCReturn(rc, rc);
15821
15822	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15823
15824	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15825	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15826	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15827	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15828	{
15829	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15830	rcStrict = VINF_SUCCESS;
15831	}
15832	return rcStrict;
15833	}
15834
15835
15836	/**
15837	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15838	*/
15839	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15840	{
15841	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15842
15843	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15844	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15845	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15846	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15847	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15848	AssertRCReturn(rc, rc);
15849
15850	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15851
15852	VMXVEXITINFO ExitInfo;
15853	RT_ZERO(ExitInfo);
15854	ExitInfo.uReason = pVmxTransient->uExitReason;
15855	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15856	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15857	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15858	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15859
15860	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15861	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15862	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15863	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15864	{
15865	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15866	rcStrict = VINF_SUCCESS;
15867	}
15868	return rcStrict;
15869	}
15870
15871
15872	/**
15873	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
15874	*/
15875	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15876	{
15877	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15878
15879	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15880	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15881	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15882	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15883	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15884	AssertRCReturn(rc, rc);
15885
15886	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15887
15888	VMXVEXITINFO ExitInfo;
15889	RT_ZERO(ExitInfo);
15890	ExitInfo.uReason = pVmxTransient->uExitReason;
15891	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15892	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15893	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15894	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15895
15896	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
15897	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15898	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15899	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15900	{
15901	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15902	rcStrict = VINF_SUCCESS;
15903	}
15904	return rcStrict;
15905	}
15906
15907	/** @} */
15908
15909	/** @name Nested-guest VM-exit handlers.
15910	* @{
15911	*/
15912	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15913	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15914	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15915
15916	/**
15917	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
15918	* Conditional VM-exit.
15919	*/
15920	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15921	{
15922	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15923
15924	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15925	AssertRCReturn(rc, rc);
15926
15927	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
15928	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
15929	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
15930
15931	/*
15932	* Make sure not to use stale/previous VM-exit instruction length since we read the
15933	* instruction length from the VMCS below only for software exceptions and privileged
15934	* software exceptions but we pass it for all exception VM-exits below.
15935	*/
15936	pVmxTransient->cbInstr = 0;
15937
15938	switch (uExtIntType)
15939	{
15940	/*
15941	* Physical NMIs:
15942	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the
15943	* host.
15944	*/
15945	case VMX_EXIT_INT_INFO_TYPE_NMI:
15946	return hmR0VmxExitHostNmi(pVCpu);
15947
15948	/*
15949	* Hardware exceptions,
15950	* Software exceptions,
15951	* Privileged software exceptions:
15952	* Figure out if the exception must be delivered to the guest or the nested-guest.
15953	*
15954	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
15955	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
15956	* length.
15957	*/
15958	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
15959	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
15960	{
15961	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15962	RT_FALL_THRU();
15963	}
15964	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
15965	{
15966	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15967	AssertRCReturn(rc, rc);
15968
15969	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
15970	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
15971	pVmxTransient->uExitIntErrorCode);
15972	if (fIntercept)
15973	{
15974	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15975	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15976	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15977	AssertRCReturn(rc, rc);
15978
15979	VMXVEXITINFO ExitInfo;
15980	RT_ZERO(ExitInfo);
15981	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15982	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15983
15984	VMXVEXITEVENTINFO ExitEventInfo;
15985	RT_ZERO(ExitInfo);
15986	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
15987	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
15988	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
15989	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
15990
15991	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
15992	}
15993
15994	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
15995	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15996
15997	/* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
15998	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
15999	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16000	return VINF_SUCCESS;
16001	}
16002
16003	/*
16004	* Software interrupts:
16005	* VM-exits cannot be caused by software interrupts.
16006	*
16007	* External interrupts:
16008	* This should only happen when "acknowledge external interrupts on VM-exit"
16009	* control is set. However, we don't set it when executing guests or
16010	* nested-guests. For nested-guests it is emulated while injecting interrupts into
16011	* the guest.
16012	*/
16013	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16014	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16015	default:
16016	{
16017	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16018	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16019	}
16020	}
16021	}
16022
16023
16024	/**
16025	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16026	* Unconditional VM-exit.
16027	*/
16028	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16029	{
16030	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16031	return IEMExecVmxVmexitTripleFault(pVCpu);
16032	}
16033
16034
16035	/**
16036	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16037	*/
16038	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16039	{
16040	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16041
16042	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16043	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16044	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16045	}
16046
16047
16048	/**
16049	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16050	*/
16051	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16052	{
16053	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16054
16055	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16056	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16057	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16058	}
16059
16060
16061	/**
16062	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16063	* Unconditional VM-exit.
16064	*/
16065	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16066	{
16067	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16068
16069	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16070	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16071	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16072	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16073	AssertRCReturn(rc, rc);
16074
16075	VMXVEXITINFO ExitInfo;
16076	RT_ZERO(ExitInfo);
16077	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16078	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16079
16080	VMXVEXITEVENTINFO ExitEventInfo;
16081	RT_ZERO(ExitInfo);
16082	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16083	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16084	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16085	}
16086
16087
16088	/**
16089	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16090	*/
16091	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16092	{
16093	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16094
16095	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16096	{
16097	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16098	AssertRCReturn(rc, rc);
16099	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16100	}
16101	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16102	}
16103
16104
16105	/**
16106	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16107	*/
16108	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16109	{
16110	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16111
16112	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16113	{
16114	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16115	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16116	AssertRCReturn(rc, rc);
16117
16118	VMXVEXITINFO ExitInfo;
16119	RT_ZERO(ExitInfo);
16120	ExitInfo.uReason = pVmxTransient->uExitReason;
16121	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16122	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16123	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16124	}
16125	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16126	}
16127
16128
16129	/**
16130	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16131	*/
16132	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16133	{
16134	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16135
16136	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16137	{
16138	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16139	AssertRCReturn(rc, rc);
16140	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16141	}
16142	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16143	}
16144
16145
16146	/**
16147	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16148	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16149	*/
16150	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16151	{
16152	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16153
16154	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16155	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16156
16157	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16158	AssertRCReturn(rc, rc);
16159
16160	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16161	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16162	uint64_t u64FieldEnc = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16163
16164	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16165	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16166	u64FieldEnc &= UINT64_C(0xffffffff);
16167
16168	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64FieldEnc))
16169	{
16170	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16171	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16172	AssertRCReturn(rc, rc);
16173
16174	VMXVEXITINFO ExitInfo;
16175	RT_ZERO(ExitInfo);
16176	ExitInfo.uReason = pVmxTransient->uExitReason;
16177	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16178	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16179	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16180	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16181	}
16182
16183	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16184	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16185	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16186	}
16187
16188
16189	/**
16190	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16191	*/
16192	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16193	{
16194	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16195
16196	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16197	{
16198	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16199	AssertRCReturn(rc, rc);
16200	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16201	}
16202
16203	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16204	}
16205
16206
16207	/**
16208	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16209	* Conditional VM-exit.
16210	*/
16211	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16212	{
16213	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16214
16215	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16216	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16217	AssertRCReturn(rc, rc);
16218
16219	VBOXSTRICTRC rcStrict;
16220	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16221	switch (uAccessType)
16222	{
16223	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16224	{
16225	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16226	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16227	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16228	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16229	if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
16230	{
16231	VMXVEXITINFO ExitInfo;
16232	RT_ZERO(ExitInfo);
16233	ExitInfo.uReason = pVmxTransient->uExitReason;
16234	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16235	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16236	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16237	}
16238	else
16239	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16240	break;
16241	}
16242
16243	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16244	{
16245	/*
16246	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16247	* CR2 reads do not cause a VM-exit.
16248	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16249	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16250	*/
16251	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16252	if ( iCrReg == 3
16253	\|\| iCrReg == 8)
16254	{
16255	static const uint32_t s_aCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16256	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16257	uint32_t const uIntercept = s_aCrXReadIntercepts[iCrReg];
16258	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16259	{
16260	VMXVEXITINFO ExitInfo;
16261	RT_ZERO(ExitInfo);
16262	ExitInfo.uReason = pVmxTransient->uExitReason;
16263	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16264	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16265	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16266	}
16267	else
16268	{
16269	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16270	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16271	}
16272	}
16273	else
16274	{
16275	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16276	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16277	}
16278	break;
16279	}
16280
16281	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16282	{
16283	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16284	Assert(pVmcsNstGst);
16285	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16286	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16287	if ( (uGstHostMask & X86_CR0_TS)
16288	&& (uReadShadow & X86_CR0_TS))
16289	{
16290	VMXVEXITINFO ExitInfo;
16291	RT_ZERO(ExitInfo);
16292	ExitInfo.uReason = pVmxTransient->uExitReason;
16293	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16294	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16295	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16296	}
16297	else
16298	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16299	break;
16300	}
16301
16302	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16303	{
16304	RTGCPTR GCPtrEffDst;
16305	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16306	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16307	if (fMemOperand)
16308	{
16309	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16310	AssertRCReturn(rc, rc);
16311	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16312	}
16313	else
16314	GCPtrEffDst = NIL_RTGCPTR;
16315
16316	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16317	{
16318	VMXVEXITINFO ExitInfo;
16319	RT_ZERO(ExitInfo);
16320	ExitInfo.uReason = pVmxTransient->uExitReason;
16321	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16322	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16323	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16324	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16325	}
16326	else
16327	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16328	break;
16329	}
16330
16331	default:
16332	{
16333	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16334	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16335	}
16336	}
16337
16338	if (rcStrict == VINF_IEM_RAISED_XCPT)
16339	{
16340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16341	rcStrict = VINF_SUCCESS;
16342	}
16343	return rcStrict;
16344	}
16345
16346
16347	/**
16348	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16349	* Conditional VM-exit.
16350	*/
16351	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16352	{
16353	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16354
16355	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16356	{
16357	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16358	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16359	AssertRCReturn(rc, rc);
16360
16361	VMXVEXITINFO ExitInfo;
16362	RT_ZERO(ExitInfo);
16363	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16364	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16365	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16366	}
16367	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16368	}
16369
16370
16371	/**
16372	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16373	* Conditional VM-exit.
16374	*/
16375	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16376	{
16377	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16378
16379	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16380	AssertRCReturn(rc, rc);
16381
16382	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16383	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16384	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16385
16386	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16387	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16388	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16389	{
16390	/*
16391	* IN/OUT instruction:
16392	* - Provides VM-exit instruction length.
16393	*
16394	* INS/OUTS instruction:
16395	* - Provides VM-exit instruction length.
16396	* - Provides Guest-linear address.
16397	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16398	*/
16399	PVM pVM = pVCpu->CTX_SUFF(pVM);
16400	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16401	AssertRCReturn(rc, rc);
16402
16403	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16404	pVmxTransient->ExitInstrInfo.u = 0;
16405	pVmxTransient->uGuestLinearAddr = 0;
16406
16407	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16408	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16409	if (fIOString)
16410	{
16411	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16412	if (fVmxInsOutsInfo)
16413	{
16414	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16415	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16416	}
16417	}
16418	AssertRCReturn(rc, rc);
16419
16420	VMXVEXITINFO ExitInfo;
16421	RT_ZERO(ExitInfo);
16422	ExitInfo.uReason = pVmxTransient->uExitReason;
16423	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16424	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16425	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16426	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16427	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16428	}
16429	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16430	}
16431
16432
16433	/**
16434	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16435	*/
16436	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16437	{
16438	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16439
16440	uint32_t fMsrpm;
16441	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16442	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16443	else
16444	fMsrpm = VMXMSRPM_EXIT_RD;
16445
16446	if (fMsrpm & VMXMSRPM_EXIT_RD)
16447	{
16448	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16449	AssertRCReturn(rc, rc);
16450	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16451	}
16452	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16453	}
16454
16455
16456	/**
16457	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16458	*/
16459	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16460	{
16461	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16462
16463	uint32_t fMsrpm;
16464	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16465	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16466	else
16467	fMsrpm = VMXMSRPM_EXIT_WR;
16468
16469	if (fMsrpm & VMXMSRPM_EXIT_WR)
16470	{
16471	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16472	AssertRCReturn(rc, rc);
16473	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16474	}
16475	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
16476	}
16477
16478
16479	/**
16480	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
16481	*/
16482	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16483	{
16484	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16485
16486	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
16487	{
16488	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16489	AssertRCReturn(rc, rc);
16490	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16491	}
16492	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
16493	}
16494
16495
16496	/**
16497	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
16498	* VM-exit.
16499	*/
16500	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16501	{
16502	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16503
16504	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
16505	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16506	}
16507
16508
16509	/**
16510	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
16511	*/
16512	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16513	{
16514	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16515
16516	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
16517	{
16518	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16519	AssertRCReturn(rc, rc);
16520	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16521	}
16522	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
16523	}
16524
16525
16526	/**
16527	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
16528	*/
16529	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16530	{
16531	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16532
16533	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
16534	* PAUSE when executing a nested-guest? If it does not, we would not need
16535	* to check for the intercepts here. Just call VM-exit... */
16536
16537	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
16538	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
16539	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
16540	{
16541	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16542	AssertRCReturn(rc, rc);
16543	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16544	}
16545	return hmR0VmxExitPause(pVCpu, pVmxTransient);
16546	}
16547
16548
16549	/**
16550	* Nested-guest VM-exit handler for when the TPR value is lowered below the
16551	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
16552	*/
16553	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16554	{
16555	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16556
16557	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
16558	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16559	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
16560	}
16561
16562
16563	/**
16564	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
16565	* VM-exit.
16566	*/
16567	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16568	{
16569	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16570
16571	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16572	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16573	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16574	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16575	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16576	AssertRCReturn(rc, rc);
16577
16578	VMXVEXITINFO ExitInfo;
16579	RT_ZERO(ExitInfo);
16580	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16581	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16582
16583	VMXVEXITEVENTINFO ExitEventInfo;
16584	RT_ZERO(ExitInfo);
16585	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16586	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16587	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
16588	}
16589
16590
16591	/**
16592	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
16593	* Conditional VM-exit.
16594	*/
16595	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16596	{
16597	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16598
16599	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
16600	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16601	AssertRCReturn(rc, rc);
16602
16603	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16604	}
16605
16606
16607	/**
16608	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
16609	* Conditional VM-exit.
16610	*/
16611	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16612	{
16613	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16614
16615	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
16616	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16617	AssertRCReturn(rc, rc);
16618
16619	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
16620	}
16621
16622
16623	/**
16624	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
16625	*/
16626	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16627	{
16628	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16629
16630	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16631	{
16632	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
16633	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16634	AssertRCReturn(rc, rc);
16635	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16636	}
16637	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
16638	}
16639
16640
16641	/**
16642	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
16643	*/
16644	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16645	{
16646	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16647
16648	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
16649	{
16650	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16651	AssertRCReturn(rc, rc);
16652	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16653	}
16654	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
16655	}
16656
16657
16658	/**
16659	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
16660	*/
16661	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16662	{
16663	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16664
16665	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16666	{
16667	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
16668	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16669	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16670	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16671	AssertRCReturn(rc, rc);
16672
16673	VMXVEXITINFO ExitInfo;
16674	RT_ZERO(ExitInfo);
16675	ExitInfo.uReason = pVmxTransient->uExitReason;
16676	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16677	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16678	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16679	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16680	}
16681	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
16682	}
16683
16684
16685	/**
16686	* Nested-guest VM-exit handler for invalid-guest state
16687	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
16688	*/
16689	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16690	{
16691	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16692
16693	/*
16694	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
16695	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
16696	* Handle it like it's in an invalid guest state of the outer guest.
16697	*
16698	* When the fast path is implemented, this should be changed to cause the corresponding
16699	* nested-guest VM-exit.
16700	*/
16701	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
16702	}
16703
16704
16705	/**
16706	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
16707	* and only provide the instruction length.
16708	*
16709	* Unconditional VM-exit.
16710	*/
16711	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16712	{
16713	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16714
16715	#ifdef VBOX_STRICT
16716	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16717	switch (pVmxTransient->uExitReason)
16718	{
16719	case VMX_EXIT_ENCLS:
16720	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
16721	break;
16722
16723	case VMX_EXIT_VMFUNC:
16724	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
16725	break;
16726	}
16727	#endif
16728
16729	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16730	AssertRCReturn(rc, rc);
16731	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16732	}
16733
16734
16735	/**
16736	* Nested-guest VM-exit handler for instructions that provide instruction length as
16737	* well as more information.
16738	*
16739	* Unconditional VM-exit.
16740	*/
16741	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16742	{
16743	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16744
16745	#ifdef VBOX_STRICT
16746	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16747	switch (pVmxTransient->uExitReason)
16748	{
16749	case VMX_EXIT_GDTR_IDTR_ACCESS:
16750	case VMX_EXIT_LDTR_TR_ACCESS:
16751	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
16752	break;
16753
16754	case VMX_EXIT_RDRAND:
16755	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
16756	break;
16757
16758	case VMX_EXIT_RDSEED:
16759	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
16760	break;
16761
16762	case VMX_EXIT_XSAVES:
16763	case VMX_EXIT_XRSTORS:
16764	/** @todo NSTVMX: Verify XSS-bitmap. */
16765	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
16766	break;
16767
16768	case VMX_EXIT_UMWAIT:
16769	case VMX_EXIT_TPAUSE:
16770	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
16771	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
16772	break;
16773	}
16774	#endif
16775
16776	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16777	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16778	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16779	AssertRCReturn(rc, rc);
16780
16781	VMXVEXITINFO ExitInfo;
16782	RT_ZERO(ExitInfo);
16783	ExitInfo.uReason = pVmxTransient->uExitReason;
16784	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16785	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16786	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16787	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16788	}
16789
16790	/** @} */
16791
16792	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16793

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79119

Download in other formats: