HMVMXR0.cpp@ 79695

Last change on this file since 79695 was 79695, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 737.3 KB

Line
1	/* $Id: HMVMXR0.cpp 79695 2019-07-11 12:41:19Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/gim.h>
38	#include <VBox/vmm/apic.h>
39	#ifdef VBOX_WITH_REM
40	# include <VBox/vmm/rem.h>
41	#endif
42	#include "HMInternal.h"
43	#include <VBox/vmm/vm.h>
44	#include <VBox/vmm/hmvmxinline.h>
45	#include "HMVMXR0.h"
46	#include "dtrace/VBoxVMM.h"
47
48	#ifdef DEBUG_ramshankar
49	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
50	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
53	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
54	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
55	# define HMVMX_ALWAYS_TRAP_PF
56	# define HMVMX_ALWAYS_FLUSH_TLB
57	# define HMVMX_ALWAYS_SWAP_EFER
58	#endif
59
60
61	/*********************************************************************************************************************************
62	* Defined Constants And Macros *
63	*********************************************************************************************************************************/
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/** @name HMVMX_READ_XXX
74	* Flags to skip redundant reads of some common VMCS fields that are not part of
75	* the guest-CPU or VCPU state but are needed while handling VM-exits.
76	*/
77	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
78	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
79	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
80	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
81	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
82	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
83	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
84	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
85	/** @} */
86
87	/**
88	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
89	* guest using hardware-assisted VMX.
90	*
91	* This excludes state like GPRs (other than RSP) which are always are
92	* swapped and restored across the world-switch and also registers like EFER,
93	* MSR which cannot be modified by the guest without causing a VM-exit.
94	*/
95	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
96	\| CPUMCTX_EXTRN_RFLAGS \
97	\| CPUMCTX_EXTRN_RSP \
98	\| CPUMCTX_EXTRN_SREG_MASK \
99	\| CPUMCTX_EXTRN_TABLE_MASK \
100	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
101	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
102	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
103	\| CPUMCTX_EXTRN_TSC_AUX \
104	\| CPUMCTX_EXTRN_OTHER_MSRS \
105	\| CPUMCTX_EXTRN_CR0 \
106	\| CPUMCTX_EXTRN_CR3 \
107	\| CPUMCTX_EXTRN_CR4 \
108	\| CPUMCTX_EXTRN_DR7 \
109	\| CPUMCTX_EXTRN_HWVIRT \
110	\| CPUMCTX_EXTRN_HM_VMX_MASK)
111
112	/**
113	* Exception bitmap mask for real-mode guests (real-on-v86).
114	*
115	* We need to intercept all exceptions manually except:
116	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
117	* due to bugs in Intel CPUs.
118	* - \#PF need not be intercepted even in real-mode if we have nested paging
119	* support.
120	*/
121	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
122	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
123	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
124	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
125	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
126	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
127	\| RT_BIT(X86_XCPT_XF))
128
129	/** Maximum VM-instruction error number. */
130	#define HMVMX_INSTR_ERROR_MAX 28
131
132	/** Profiling macro. */
133	#ifdef HM_PROFILE_EXIT_DISPATCH
134	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
135	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
136	#else
137	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
138	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
139	#endif
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
151
152	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
153	* context. */
154	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
155	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
156	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
157
158	/** Log the VM-exit reason with an easily visible marker to identify it in a
159	* potential sea of logging data. */
160	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
161	do { \
162	Log4(("VM-exit: vcpu[%RU32] reason=%#x -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", \
163	(a_pVCpu)->idCpu, (a_uExitReason))); \
164	} while (0) \
165
166
167	/*********************************************************************************************************************************
168	* Structures and Typedefs *
169	*********************************************************************************************************************************/
170	/**
171	* VMX per-VCPU transient state.
172	*
173	* A state structure for holding miscellaneous information across
174	* VMX non-root operation and restored after the transition.
175	*/
176	typedef struct VMXTRANSIENT
177	{
178	/** The host's rflags/eflags. */
179	RTCCUINTREG fEFlags;
180	#if HC_ARCH_BITS == 32
181	uint32_t u32Alignment0;
182	#endif
183	/** The guest's TPR value used for TPR shadowing. */
184	uint8_t u8GuestTpr;
185	/** Alignment. */
186	uint8_t abAlignment0[7];
187
188	/** The basic VM-exit reason. */
189	uint16_t uExitReason;
190	/** Alignment. */
191	uint16_t u16Alignment0;
192	/** The VM-exit interruption error code. */
193	uint32_t uExitIntErrorCode;
194	/** The VM-exit exit code qualification. */
195	uint64_t uExitQual;
196	/** The Guest-linear address. */
197	uint64_t uGuestLinearAddr;
198
199	/** The VM-exit interruption-information field. */
200	uint32_t uExitIntInfo;
201	/** The VM-exit instruction-length field. */
202	uint32_t cbInstr;
203	/** The VM-exit instruction-information field. */
204	VMXEXITINSTRINFO ExitInstrInfo;
205	/** Whether the VM-entry failed or not. */
206	bool fVMEntryFailed;
207	/** Whether we are currently executing a nested-guest. */
208	bool fIsNestedGuest;
209	/** Alignment. */
210	uint8_t abAlignment1[2];
211
212	/** The VM-entry interruption-information field. */
213	uint32_t uEntryIntInfo;
214	/** The VM-entry exception error code field. */
215	uint32_t uEntryXcptErrorCode;
216	/** The VM-entry instruction length field. */
217	uint32_t cbEntryInstr;
218
219	/** IDT-vectoring information field. */
220	uint32_t uIdtVectoringInfo;
221	/** IDT-vectoring error code. */
222	uint32_t uIdtVectoringErrorCode;
223
224	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
225	uint32_t fVmcsFieldsRead;
226
227	/** Whether the guest debug state was active at the time of VM-exit. */
228	bool fWasGuestDebugStateActive;
229	/** Whether the hyper debug state was active at the time of VM-exit. */
230	bool fWasHyperDebugStateActive;
231	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
232	bool fUpdatedTscOffsettingAndPreemptTimer;
233	/** Whether the VM-exit was caused by a page-fault during delivery of a
234	* contributory exception or a page-fault. */
235	bool fVectoringDoublePF;
236	/** Whether the VM-exit was caused by a page-fault during delivery of an
237	* external interrupt or NMI. */
238	bool fVectoringPF;
239	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
240	* area after VM-exit. */
241	bool fRemoveTscAuxMsr;
242	bool afAlignment0[2];
243
244	/** The VMCS info. object. */
245	PVMXVMCSINFO pVmcsInfo;
246	} VMXTRANSIENT;
247	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
248	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
249	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
250	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
251	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
252	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
253	/** Pointer to VMX transient state. */
254	typedef VMXTRANSIENT *PVMXTRANSIENT;
255	/** Pointer to a const VMX transient state. */
256	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
257
258	/**
259	* Memory operand read or write access.
260	*/
261	typedef enum VMXMEMACCESS
262	{
263	VMXMEMACCESS_READ = 0,
264	VMXMEMACCESS_WRITE = 1
265	} VMXMEMACCESS;
266
267	/**
268	* VMX VM-exit handler.
269	*
270	* @returns Strict VBox status code (i.e. informational status codes too).
271	* @param pVCpu The cross context virtual CPU structure.
272	* @param pVmxTransient The VMX-transient structure.
273	*/
274	#ifndef HMVMX_USE_FUNCTION_TABLE
275	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
276	#else
277	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
278	/** Pointer to VM-exit handler. */
279	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
280	#endif
281
282	/**
283	* VMX VM-exit handler, non-strict status code.
284	*
285	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
286	*
287	* @returns VBox status code, no informational status code returned.
288	* @param pVCpu The cross context virtual CPU structure.
289	* @param pVmxTransient The VMX-transient structure.
290	*
291	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
292	* use of that status code will be replaced with VINF_EM_SOMETHING
293	* later when switching over to IEM.
294	*/
295	#ifndef HMVMX_USE_FUNCTION_TABLE
296	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
297	#else
298	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
299	#endif
300
301
302	/*********************************************************************************************************************************
303	* Internal Functions *
304	*********************************************************************************************************************************/
305	#ifndef HMVMX_USE_FUNCTION_TABLE
306	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
307	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
308	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
309	#else
310	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
311	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
312	#endif
313	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
314	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
315	#endif
316
317	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
318	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
319	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
320	#endif
321
322	/** @name VM-exit handler prototypes.
323	* @{
324	*/
325	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
326	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
327	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
328	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
329	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
330	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
331	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
332	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
333	static FNVMXEXITHANDLER hmR0VmxExitHlt;
334	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
335	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
336	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
337	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
338	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
339	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
340	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
341	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
342	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
343	static FNVMXEXITHANDLER hmR0VmxExitVmread;
344	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
345	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
346	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
347	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
348	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
349	#endif
350	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
351	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
352	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
353	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
354	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
355	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
356	static FNVMXEXITHANDLER hmR0VmxExitMwait;
357	static FNVMXEXITHANDLER hmR0VmxExitMtf;
358	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
359	static FNVMXEXITHANDLER hmR0VmxExitPause;
360	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
361	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
362	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
363	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
364	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
365	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
367	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
368	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
371	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
372	/** @} */
373
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	/** @name Nested-guest VM-exit handler prototypes.
376	* @{
377	*/
378	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
379	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
380	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
381	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
382	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
383	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
384	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
385	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
386	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
387	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
388	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
389	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
390	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
391	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
392	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
393	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
394	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
395	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
396	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
398	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
399	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
400	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
405	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
406	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
407	/** @} */
408	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
409
410
411	/*********************************************************************************************************************************
412	* Global Variables *
413	*********************************************************************************************************************************/
414	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
415	/**
416	* Array of all VMCS fields.
417	* Any fields added to the VT-x spec. should be added here.
418	*
419	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
420	* of nested-guests.
421	*/
422	static const uint32_t g_aVmcsFields[] =
423	{
424	/* 16-bit control fields. */
425	VMX_VMCS16_VPID,
426	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
427	VMX_VMCS16_EPTP_INDEX,
428
429	/* 16-bit guest-state fields. */
430	VMX_VMCS16_GUEST_ES_SEL,
431	VMX_VMCS16_GUEST_CS_SEL,
432	VMX_VMCS16_GUEST_SS_SEL,
433	VMX_VMCS16_GUEST_DS_SEL,
434	VMX_VMCS16_GUEST_FS_SEL,
435	VMX_VMCS16_GUEST_GS_SEL,
436	VMX_VMCS16_GUEST_LDTR_SEL,
437	VMX_VMCS16_GUEST_TR_SEL,
438	VMX_VMCS16_GUEST_INTR_STATUS,
439	VMX_VMCS16_GUEST_PML_INDEX,
440
441	/* 16-bits host-state fields. */
442	VMX_VMCS16_HOST_ES_SEL,
443	VMX_VMCS16_HOST_CS_SEL,
444	VMX_VMCS16_HOST_SS_SEL,
445	VMX_VMCS16_HOST_DS_SEL,
446	VMX_VMCS16_HOST_FS_SEL,
447	VMX_VMCS16_HOST_GS_SEL,
448	VMX_VMCS16_HOST_TR_SEL,
449
450	/* 64-bit control fields. */
451	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
452	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
453	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
454	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
455	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
456	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
457	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
458	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
459	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
460	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
461	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
462	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
463	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
464	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
465	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
466	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
467	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
468	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
469	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
470	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
471	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
472	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
473	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
474	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
475	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
476	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
477	VMX_VMCS64_CTRL_EPTP_FULL,
478	VMX_VMCS64_CTRL_EPTP_HIGH,
479	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
480	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
481	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
482	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
483	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
484	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
485	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
486	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
487	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
488	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
489	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
490	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
491	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
492	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
493	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
494	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
495	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
496	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
497	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
498	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
499	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
500	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
501
502	/* 64-bit read-only data fields. */
503	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
504	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
505
506	/* 64-bit guest-state fields. */
507	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
508	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
509	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
510	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
511	VMX_VMCS64_GUEST_PAT_FULL,
512	VMX_VMCS64_GUEST_PAT_HIGH,
513	VMX_VMCS64_GUEST_EFER_FULL,
514	VMX_VMCS64_GUEST_EFER_HIGH,
515	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
516	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
517	VMX_VMCS64_GUEST_PDPTE0_FULL,
518	VMX_VMCS64_GUEST_PDPTE0_HIGH,
519	VMX_VMCS64_GUEST_PDPTE1_FULL,
520	VMX_VMCS64_GUEST_PDPTE1_HIGH,
521	VMX_VMCS64_GUEST_PDPTE2_FULL,
522	VMX_VMCS64_GUEST_PDPTE2_HIGH,
523	VMX_VMCS64_GUEST_PDPTE3_FULL,
524	VMX_VMCS64_GUEST_PDPTE3_HIGH,
525	VMX_VMCS64_GUEST_BNDCFGS_FULL,
526	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
527
528	/* 64-bit host-state fields. */
529	VMX_VMCS64_HOST_PAT_FULL,
530	VMX_VMCS64_HOST_PAT_HIGH,
531	VMX_VMCS64_HOST_EFER_FULL,
532	VMX_VMCS64_HOST_EFER_HIGH,
533	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
534	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
535
536	/* 32-bit control fields. */
537	VMX_VMCS32_CTRL_PIN_EXEC,
538	VMX_VMCS32_CTRL_PROC_EXEC,
539	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
540	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
541	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
542	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
543	VMX_VMCS32_CTRL_EXIT,
544	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
545	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
546	VMX_VMCS32_CTRL_ENTRY,
547	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
548	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
549	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
550	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
551	VMX_VMCS32_CTRL_TPR_THRESHOLD,
552	VMX_VMCS32_CTRL_PROC_EXEC2,
553	VMX_VMCS32_CTRL_PLE_GAP,
554	VMX_VMCS32_CTRL_PLE_WINDOW,
555
556	/* 32-bits read-only fields. */
557	VMX_VMCS32_RO_VM_INSTR_ERROR,
558	VMX_VMCS32_RO_EXIT_REASON,
559	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
560	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
561	VMX_VMCS32_RO_IDT_VECTORING_INFO,
562	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
563	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
564	VMX_VMCS32_RO_EXIT_INSTR_INFO,
565
566	/* 32-bit guest-state fields. */
567	VMX_VMCS32_GUEST_ES_LIMIT,
568	VMX_VMCS32_GUEST_CS_LIMIT,
569	VMX_VMCS32_GUEST_SS_LIMIT,
570	VMX_VMCS32_GUEST_DS_LIMIT,
571	VMX_VMCS32_GUEST_FS_LIMIT,
572	VMX_VMCS32_GUEST_GS_LIMIT,
573	VMX_VMCS32_GUEST_LDTR_LIMIT,
574	VMX_VMCS32_GUEST_TR_LIMIT,
575	VMX_VMCS32_GUEST_GDTR_LIMIT,
576	VMX_VMCS32_GUEST_IDTR_LIMIT,
577	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
578	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
579	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
580	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
581	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
582	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
583	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
584	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
585	VMX_VMCS32_GUEST_INT_STATE,
586	VMX_VMCS32_GUEST_ACTIVITY_STATE,
587	VMX_VMCS32_GUEST_SMBASE,
588	VMX_VMCS32_GUEST_SYSENTER_CS,
589	VMX_VMCS32_PREEMPT_TIMER_VALUE,
590
591	/* 32-bit host-state fields. */
592	VMX_VMCS32_HOST_SYSENTER_CS,
593
594	/* Natural-width control fields. */
595	VMX_VMCS_CTRL_CR0_MASK,
596	VMX_VMCS_CTRL_CR4_MASK,
597	VMX_VMCS_CTRL_CR0_READ_SHADOW,
598	VMX_VMCS_CTRL_CR4_READ_SHADOW,
599	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
600	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
601	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
602	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
603
604	/* Natural-width read-only data fields. */
605	VMX_VMCS_RO_EXIT_QUALIFICATION,
606	VMX_VMCS_RO_IO_RCX,
607	VMX_VMCS_RO_IO_RSI,
608	VMX_VMCS_RO_IO_RDI,
609	VMX_VMCS_RO_IO_RIP,
610	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
611
612	/* Natural-width guest-state field */
613	VMX_VMCS_GUEST_CR0,
614	VMX_VMCS_GUEST_CR3,
615	VMX_VMCS_GUEST_CR4,
616	VMX_VMCS_GUEST_ES_BASE,
617	VMX_VMCS_GUEST_CS_BASE,
618	VMX_VMCS_GUEST_SS_BASE,
619	VMX_VMCS_GUEST_DS_BASE,
620	VMX_VMCS_GUEST_FS_BASE,
621	VMX_VMCS_GUEST_GS_BASE,
622	VMX_VMCS_GUEST_LDTR_BASE,
623	VMX_VMCS_GUEST_TR_BASE,
624	VMX_VMCS_GUEST_GDTR_BASE,
625	VMX_VMCS_GUEST_IDTR_BASE,
626	VMX_VMCS_GUEST_DR7,
627	VMX_VMCS_GUEST_RSP,
628	VMX_VMCS_GUEST_RIP,
629	VMX_VMCS_GUEST_RFLAGS,
630	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
631	VMX_VMCS_GUEST_SYSENTER_ESP,
632	VMX_VMCS_GUEST_SYSENTER_EIP,
633
634	/* Natural-width host-state fields */
635	VMX_VMCS_HOST_CR0,
636	VMX_VMCS_HOST_CR3,
637	VMX_VMCS_HOST_CR4,
638	VMX_VMCS_HOST_FS_BASE,
639	VMX_VMCS_HOST_GS_BASE,
640	VMX_VMCS_HOST_TR_BASE,
641	VMX_VMCS_HOST_GDTR_BASE,
642	VMX_VMCS_HOST_IDTR_BASE,
643	VMX_VMCS_HOST_SYSENTER_ESP,
644	VMX_VMCS_HOST_SYSENTER_EIP,
645	VMX_VMCS_HOST_RSP,
646	VMX_VMCS_HOST_RIP
647	};
648	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
649
650	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
651	static const uint32_t g_aVmcsCacheSegBase[] =
652	{
653	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
654	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
655	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
656	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
657	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
658	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
659	};
660	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
661	#endif
662	static const uint32_t g_aVmcsSegBase[] =
663	{
664	VMX_VMCS_GUEST_ES_BASE,
665	VMX_VMCS_GUEST_CS_BASE,
666	VMX_VMCS_GUEST_SS_BASE,
667	VMX_VMCS_GUEST_DS_BASE,
668	VMX_VMCS_GUEST_FS_BASE,
669	VMX_VMCS_GUEST_GS_BASE
670	};
671	static const uint32_t g_aVmcsSegSel[] =
672	{
673	VMX_VMCS16_GUEST_ES_SEL,
674	VMX_VMCS16_GUEST_CS_SEL,
675	VMX_VMCS16_GUEST_SS_SEL,
676	VMX_VMCS16_GUEST_DS_SEL,
677	VMX_VMCS16_GUEST_FS_SEL,
678	VMX_VMCS16_GUEST_GS_SEL
679	};
680	static const uint32_t g_aVmcsSegLimit[] =
681	{
682	VMX_VMCS32_GUEST_ES_LIMIT,
683	VMX_VMCS32_GUEST_CS_LIMIT,
684	VMX_VMCS32_GUEST_SS_LIMIT,
685	VMX_VMCS32_GUEST_DS_LIMIT,
686	VMX_VMCS32_GUEST_FS_LIMIT,
687	VMX_VMCS32_GUEST_GS_LIMIT
688	};
689	static const uint32_t g_aVmcsSegAttr[] =
690	{
691	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
692	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
693	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
694	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
695	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
696	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
697	};
698	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
699	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
700	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
701	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
702
703	#ifdef HMVMX_USE_FUNCTION_TABLE
704	/**
705	* VMX_EXIT dispatch table.
706	*/
707	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
708	{
709	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
710	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
711	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
712	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
713	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
714	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
715	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
716	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
717	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
718	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
719	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
720	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
721	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
722	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
723	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
724	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
725	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
726	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
727	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
728	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
729	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
730	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
731	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
732	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
733	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
734	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
735	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
736	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
737	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
738	#else
739	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
740	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
741	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
742	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
743	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
744	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
745	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
746	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
747	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
748	#endif
749	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
750	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
751	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
752	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
753	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
754	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
755	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
756	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
757	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
758	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
759	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
760	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
761	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
762	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
763	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
765	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
766	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
767	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
768	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
769	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
770	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
771	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
772	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
773	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
774	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
775	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
776	#else
777	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
778	#endif
779	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
780	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
781	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
782	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
783	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
784	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
785	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
786	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
787	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
788	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
789	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
790	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
791	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
792	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
793	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
794	};
795	#endif /* HMVMX_USE_FUNCTION_TABLE */
796
797	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
798	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
799	{
800	/* 0 */ "(Not Used)",
801	/* 1 */ "VMCALL executed in VMX root operation.",
802	/* 2 */ "VMCLEAR with invalid physical address.",
803	/* 3 */ "VMCLEAR with VMXON pointer.",
804	/* 4 */ "VMLAUNCH with non-clear VMCS.",
805	/* 5 */ "VMRESUME with non-launched VMCS.",
806	/* 6 */ "VMRESUME after VMXOFF",
807	/* 7 */ "VM-entry with invalid control fields.",
808	/* 8 */ "VM-entry with invalid host state fields.",
809	/* 9 */ "VMPTRLD with invalid physical address.",
810	/* 10 */ "VMPTRLD with VMXON pointer.",
811	/* 11 */ "VMPTRLD with incorrect revision identifier.",
812	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
813	/* 13 */ "VMWRITE to read-only VMCS component.",
814	/* 14 */ "(Not Used)",
815	/* 15 */ "VMXON executed in VMX root operation.",
816	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
817	/* 17 */ "VM-entry with non-launched executing VMCS.",
818	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
819	/* 19 */ "VMCALL with non-clear VMCS.",
820	/* 20 */ "VMCALL with invalid VM-exit control fields.",
821	/* 21 */ "(Not Used)",
822	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
823	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
824	/* 24 */ "VMCALL with invalid SMM-monitor features.",
825	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
826	/* 26 */ "VM-entry with events blocked by MOV SS.",
827	/* 27 */ "(Not Used)",
828	/* 28 */ "Invalid operand to INVEPT/INVVPID."
829	};
830	#endif /* VBOX_STRICT && LOG_ENABLED */
831
832
833	/**
834	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
835	*
836	* Any bit set in this mask is owned by the host/hypervisor and would cause a
837	* VM-exit when modified by the guest.
838	*
839	* @returns The static CR0 guest/host mask.
840	* @param pVCpu The cross context virtual CPU structure.
841	*/
842	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
843	{
844	/*
845	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
846	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
847	*/
848	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
849	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
850	* and @bugref{6944}. */
851	PVM pVM = pVCpu->CTX_SUFF(pVM);
852	return ( X86_CR0_PE
853	\| X86_CR0_NE
854	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
855	\| X86_CR0_PG
856	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
857	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
858	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
859	}
860
861
862	/**
863	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
864	*
865	* Any bit set in this mask is owned by the host/hypervisor and would cause a
866	* VM-exit when modified by the guest.
867	*
868	* @returns The static CR4 guest/host mask.
869	* @param pVCpu The cross context virtual CPU structure.
870	*/
871	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
872	{
873	/*
874	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
875	* these bits are reserved on hardware that does not support them. Since the
876	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
877	* these bits and handle it depending on whether we expose them to the guest.
878	*/
879	PVM pVM = pVCpu->CTX_SUFF(pVM);
880	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
881	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
882	return ( X86_CR4_VMXE
883	\| X86_CR4_VME
884	\| X86_CR4_PAE
885	\| X86_CR4_PGE
886	\| X86_CR4_PSE
887	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
888	\| (fPcid ? X86_CR4_PCIDE : 0));
889	}
890
891
892	/**
893	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
894	* area.
895	*
896	* @returns @c true if it's different, @c false otherwise.
897	* @param pVmcsInfo The VMCS info. object.
898	*/
899	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
900	{
901	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
902	&& pVmcsInfo->pvGuestMsrStore);
903	}
904
905
906	/**
907	* Checks whether one of the given Pin-based VM-execution controls are set.
908	*
909	* @returns @c true if set, @c false otherwise.
910	* @param pVCpu The cross context virtual CPU structure.
911	* @param pVmxTransient The VMX-transient structure.
912	* @param uPinCtls The Pin-based VM-execution controls to check.
913	*
914	* @remarks This will not check merged controls when executing a nested-guest
915	* but the original control specified by the guest hypervisor.
916	*/
917	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
918	{
919	if (!pVmxTransient->fIsNestedGuest)
920	{
921	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
922	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
923	}
924	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
925	}
926
927
928	/**
929	* Sets the given Processor-based VM-execution controls.
930	*
931	* @param pVmxTransient The VMX-transient structure.
932	* @param uProcCtls The Processor-based VM-execution controls to set.
933	*/
934	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
935	{
936	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
937	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
938	{
939	pVmcsInfo->u32ProcCtls \|= uProcCtls;
940	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
941	AssertRC(rc);
942	}
943	}
944
945
946	/**
947	* Removes the given Processor-based VM-execution controls.
948	*
949	* @param pVCpu The cross context virtual CPU structure.
950	* @param pVmxTransient The VMX-transient structure.
951	* @param uProcCtls The Processor-based VM-execution controls to remove.
952	*
953	* @remarks When executing a nested-guest, this will not remove any of the specified
954	* controls if the guest hypervisor has set any one of them.
955	*/
956	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
957	{
958	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
959	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
960	? true
961	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT);
962	#else
963	NOREF(pVCpu);
964	bool const fRemoveCtls = true;
965	#endif
966	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
967	if ( fRemoveCtls
968	&& (pVmcsInfo->u32ProcCtls & uProcCtls))
969	{
970	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
971	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
972	AssertRC(rc);
973	}
974	}
975
976
977	/**
978	* Sets the TSC offset for the current VMCS.
979	*
980	* @param pVCpu The cross context virtual CPU structure.
981	* @param uTscOffset The TSC offset to set.
982	* @param pVmcsInfo The VMCS info. object.
983	*/
984	static void hmR0VmxSetTscOffsetVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
985	{
986	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
987	if (pVmcsInfo->u64TscOffset != uTscOffset)
988	{
989	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
990	AssertRC(rc);
991	pVmcsInfo->u64TscOffset = uTscOffset;
992	}
993	}
994
995
996	/**
997	* Adds one or more exceptions to the exception bitmap and commits it to the current
998	* VMCS.
999	*
1000	* @returns VBox status code.
1001	* @param pVmxTransient The VMX-transient structure.
1002	* @param uXcptMask The exception(s) to add.
1003	*/
1004	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1005	{
1006	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1007	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1008	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1009	{
1010	uXcptBitmap \|= uXcptMask;
1011	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1012	AssertRCReturn(rc, rc);
1013	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1014	}
1015	return VINF_SUCCESS;
1016	}
1017
1018
1019	/**
1020	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1021	*
1022	* @returns VBox status code.
1023	* @param pVmxTransient The VMX-transient structure.
1024	* @param uXcpt The exception to add.
1025	*/
1026	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1027	{
1028	Assert(uXcpt <= X86_XCPT_LAST);
1029	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1030	}
1031
1032
1033	/**
1034	* Remove one or more exceptions from the exception bitmap and commits it to the
1035	* current VMCS.
1036	*
1037	* This takes care of not removing the exception intercept if a nested-guest
1038	* requires the exception to be intercepted.
1039	*
1040	* @returns VBox status code.
1041	* @param pVCpu The cross context virtual CPU structure.
1042	* @param pVmxTransient The VMX-transient structure.
1043	* @param uXcptMask The exception(s) to remove.
1044	*/
1045	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1046	{
1047	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1048	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1049	if (u32XcptBitmap & uXcptMask)
1050	{
1051	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1052	if (!pVmxTransient->fIsNestedGuest)
1053	{ /* likely */ }
1054	else
1055	{
1056	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1057	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1058	}
1059	#endif
1060	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1061	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1062	\| RT_BIT(X86_XCPT_DE)
1063	\| RT_BIT(X86_XCPT_NM)
1064	\| RT_BIT(X86_XCPT_TS)
1065	\| RT_BIT(X86_XCPT_UD)
1066	\| RT_BIT(X86_XCPT_NP)
1067	\| RT_BIT(X86_XCPT_SS)
1068	\| RT_BIT(X86_XCPT_GP)
1069	\| RT_BIT(X86_XCPT_PF)
1070	\| RT_BIT(X86_XCPT_MF));
1071	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1072	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1073	#endif
1074	if (uXcptMask)
1075	{
1076	/* Validate we are not removing any essential exception intercepts. */
1077	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1078	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1079	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1080
1081	/* Remove it from the exception bitmap. */
1082	u32XcptBitmap &= ~uXcptMask;
1083
1084	/* Commit and update the cache if necessary. */
1085	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1086	{
1087	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1088	AssertRCReturn(rc, rc);
1089	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1090	}
1091	}
1092	}
1093	return VINF_SUCCESS;
1094	}
1095
1096
1097	/**
1098	* Remove an exceptions from the exception bitmap and commits it to the current
1099	* VMCS.
1100	*
1101	* @returns VBox status code.
1102	* @param pVCpu The cross context virtual CPU structure.
1103	* @param pVmxTransient The VMX-transient structure.
1104	* @param uXcpt The exception to remove.
1105	*/
1106	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1107	{
1108	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1109	}
1110
1111
1112	/**
1113	* Loads the VMCS specified by the VMCS info. object.
1114	*
1115	* @returns VBox status code.
1116	* @param pVmcsInfo The VMCS info. object.
1117	*
1118	* @remarks Can be called with interrupts disabled.
1119	*/
1120	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1121	{
1122	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1123	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1124
1125	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1126	if (RT_SUCCESS(rc))
1127	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1128	return rc;
1129	}
1130
1131
1132	/**
1133	* Clears the VMCS specified by the VMCS info. object.
1134	*
1135	* @returns VBox status code.
1136	* @param pVmcsInfo The VMCS info. object.
1137	*
1138	* @remarks Can be called with interrupts disabled.
1139	*/
1140	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1141	{
1142	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1143	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1144
1145	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1146	if (RT_SUCCESS(rc))
1147	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1148	return rc;
1149	}
1150
1151
1152	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1153	/**
1154	* Loads the shadow VMCS specified by the VMCS info. object.
1155	*
1156	* @returns VBox status code.
1157	* @param pVmcsInfo The VMCS info. object.
1158	*
1159	* @remarks Can be called with interrupts disabled.
1160	*/
1161	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1162	{
1163	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1164	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1165
1166	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1167	if (RT_SUCCESS(rc))
1168	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1169	return rc;
1170	}
1171
1172
1173	/**
1174	* Clears the shadow VMCS specified by the VMCS info. object.
1175	*
1176	* @returns VBox status code.
1177	* @param pVmcsInfo The VMCS info. object.
1178	*
1179	* @remarks Can be called with interrupts disabled.
1180	*/
1181	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1182	{
1183	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1184	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1185
1186	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1187	if (RT_SUCCESS(rc))
1188	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1189	return rc;
1190	}
1191
1192
1193	/**
1194	* Switches from and to the specified VMCSes.
1195	*
1196	* @returns VBox status code.
1197	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1198	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1199	*
1200	* @remarks Called with interrupts disabled.
1201	*/
1202	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1203	{
1204	/*
1205	* Clear the VMCS we are switching out if it has not already been cleared.
1206	* This will sync any CPU internal data back to the VMCS.
1207	*/
1208	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1209	{
1210	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1211	if (RT_SUCCESS(rc))
1212	{
1213	/*
1214	* The shadow VMCS, if any, would not be active at this point since we
1215	* would have cleared it while importing the virtual hardware-virtualization
1216	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1217	* clear the shadow VMCS here, just assert for safety.
1218	*/
1219	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1220	}
1221	else
1222	return rc;
1223	}
1224
1225	/*
1226	* Clear the VMCS we are switching to if it has not already been cleared.
1227	* This will initialize the VMCS launch state to "clear" required for loading it.
1228	*
1229	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1230	*/
1231	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1232	{
1233	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1234	if (RT_SUCCESS(rc))
1235	{ /* likely */ }
1236	else
1237	return rc;
1238	}
1239
1240	/*
1241	* Finally, load the VMCS we are switching to.
1242	*/
1243	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1244	}
1245
1246
1247	/**
1248	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1249	* caller.
1250	*
1251	* @returns VBox status code.
1252	* @param pVCpu The cross context virtual CPU structure.
1253	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1254	* true) or guest VMCS (pass false).
1255	*/
1256	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1257	{
1258	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1259	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1260
1261	PVMXVMCSINFO pVmcsInfoFrom;
1262	PVMXVMCSINFO pVmcsInfoTo;
1263	if (fSwitchToNstGstVmcs)
1264	{
1265	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1266	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1267	}
1268	else
1269	{
1270	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1271	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1272	}
1273
1274	/*
1275	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1276	* preemption hook code path acquires the current VMCS.
1277	*/
1278	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1279
1280	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1281	if (RT_SUCCESS(rc))
1282	{
1283	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1284
1285	/*
1286	* If we are switching to a VMCS that was executed on a different host CPU or was
1287	* never executed before, flag that we need to export the host state before executing
1288	* guest/nested-guest code using hardware-assisted VMX.
1289	*
1290	* This could probably be done in a preemptible context since the preemption hook
1291	* will flag the necessary change in host context. However, since preemption is
1292	* already disabled and to avoid making assumptions about host specific code in
1293	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1294	* disabled.
1295	*/
1296	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1297	{ /* likely */ }
1298	else
1299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1300
1301	ASMSetFlags(fEFlags);
1302
1303	/*
1304	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1305	* flag that we need to update the host MSR values there. Even if we decide in the
1306	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1307	* if its content differs, we would have to update the host MSRs anyway.
1308	*/
1309	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1310	}
1311	else
1312	ASMSetFlags(fEFlags);
1313	return rc;
1314	}
1315	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1316
1317
1318	/**
1319	* Updates the VM's last error record.
1320	*
1321	* If there was a VMX instruction error, reads the error data from the VMCS and
1322	* updates VCPU's last error record as well.
1323	*
1324	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1325	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1326	* VERR_VMX_INVALID_VMCS_FIELD.
1327	* @param rc The error code.
1328	*/
1329	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1330	{
1331	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1332	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1333	{
1334	AssertPtrReturnVoid(pVCpu);
1335	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1336	}
1337	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1338	}
1339
1340
1341	#ifdef VBOX_STRICT
1342	/**
1343	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1344	* transient structure.
1345	*
1346	* @returns VBox status code.
1347	* @param pVmxTransient The VMX-transient structure.
1348	*
1349	* @remarks No-long-jump zone!!!
1350	*/
1351	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1352	{
1353	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1354	AssertRCReturn(rc, rc);
1355	return VINF_SUCCESS;
1356	}
1357
1358
1359	/**
1360	* Reads the VM-entry exception error code field from the VMCS into
1361	* the VMX transient structure.
1362	*
1363	* @returns VBox status code.
1364	* @param pVmxTransient The VMX-transient structure.
1365	*
1366	* @remarks No-long-jump zone!!!
1367	*/
1368	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1369	{
1370	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1371	AssertRCReturn(rc, rc);
1372	return VINF_SUCCESS;
1373	}
1374
1375
1376	/**
1377	* Reads the VM-entry exception error code field from the VMCS into
1378	* the VMX transient structure.
1379	*
1380	* @returns VBox status code.
1381	* @param pVmxTransient The VMX-transient structure.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1386	{
1387	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1388	AssertRCReturn(rc, rc);
1389	return VINF_SUCCESS;
1390	}
1391	#endif /* VBOX_STRICT */
1392
1393
1394	/**
1395	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1396	* transient structure.
1397	*
1398	* @returns VBox status code.
1399	* @param pVmxTransient The VMX-transient structure.
1400	*/
1401	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1402	{
1403	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1404	{
1405	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1406	AssertRCReturn(rc,rc);
1407	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1408	}
1409	return VINF_SUCCESS;
1410	}
1411
1412
1413	/**
1414	* Reads the VM-exit interruption error code from the VMCS into the VMX
1415	* transient structure.
1416	*
1417	* @returns VBox status code.
1418	* @param pVmxTransient The VMX-transient structure.
1419	*/
1420	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1421	{
1422	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1423	{
1424	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1425	AssertRCReturn(rc, rc);
1426	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1427	}
1428	return VINF_SUCCESS;
1429	}
1430
1431
1432	/**
1433	* Reads the VM-exit instruction length field from the VMCS into the VMX
1434	* transient structure.
1435	*
1436	* @returns VBox status code.
1437	* @param pVmxTransient The VMX-transient structure.
1438	*/
1439	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1440	{
1441	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1442	{
1443	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1444	AssertRCReturn(rc, rc);
1445	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1446	}
1447	return VINF_SUCCESS;
1448	}
1449
1450
1451	/**
1452	* Reads the VM-exit instruction-information field from the VMCS into
1453	* the VMX transient structure.
1454	*
1455	* @returns VBox status code.
1456	* @param pVmxTransient The VMX-transient structure.
1457	*/
1458	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1459	{
1460	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1461	{
1462	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1463	AssertRCReturn(rc, rc);
1464	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1465	}
1466	return VINF_SUCCESS;
1467	}
1468
1469
1470	/**
1471	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1472	*
1473	* @returns VBox status code.
1474	* @param pVCpu The cross context virtual CPU structure of the
1475	* calling EMT. (Required for the VMCS cache case.)
1476	* @param pVmxTransient The VMX-transient structure.
1477	*/
1478	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1479	{
1480	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1481	{
1482	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1483	AssertRCReturn(rc, rc);
1484	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1485	}
1486	return VINF_SUCCESS;
1487	}
1488
1489
1490	/**
1491	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1492	*
1493	* @returns VBox status code.
1494	* @param pVCpu The cross context virtual CPU structure of the
1495	* calling EMT. (Required for the VMCS cache case.)
1496	* @param pVmxTransient The VMX-transient structure.
1497	*/
1498	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1499	{
1500	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1501	{
1502	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1503	AssertRCReturn(rc, rc);
1504	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1505	}
1506	return VINF_SUCCESS;
1507	}
1508
1509
1510	/**
1511	* Reads the IDT-vectoring information field from the VMCS into the VMX
1512	* transient structure.
1513	*
1514	* @returns VBox status code.
1515	* @param pVmxTransient The VMX-transient structure.
1516	*
1517	* @remarks No-long-jump zone!!!
1518	*/
1519	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1520	{
1521	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1522	{
1523	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1524	AssertRCReturn(rc, rc);
1525	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1526	}
1527	return VINF_SUCCESS;
1528	}
1529
1530
1531	/**
1532	* Reads the IDT-vectoring error code from the VMCS into the VMX
1533	* transient structure.
1534	*
1535	* @returns VBox status code.
1536	* @param pVmxTransient The VMX-transient structure.
1537	*/
1538	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1539	{
1540	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1541	{
1542	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1543	AssertRCReturn(rc, rc);
1544	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1545	}
1546	return VINF_SUCCESS;
1547	}
1548
1549
1550	/**
1551	* Enters VMX root mode operation on the current CPU.
1552	*
1553	* @returns VBox status code.
1554	* @param pVM The cross context VM structure. Can be
1555	* NULL, after a resume.
1556	* @param HCPhysCpuPage Physical address of the VMXON region.
1557	* @param pvCpuPage Pointer to the VMXON region.
1558	*/
1559	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1560	{
1561	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1562	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1563	Assert(pvCpuPage);
1564	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1565
1566	if (pVM)
1567	{
1568	/* Write the VMCS revision identifier to the VMXON region. */
1569	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1570	}
1571
1572	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1573	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1574
1575	/* Enable the VMX bit in CR4 if necessary. */
1576	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1577
1578	/* Enter VMX root mode. */
1579	int rc = VMXEnable(HCPhysCpuPage);
1580	if (RT_FAILURE(rc))
1581	{
1582	if (!(uOldCr4 & X86_CR4_VMXE))
1583	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1584
1585	if (pVM)
1586	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1587	}
1588
1589	/* Restore interrupts. */
1590	ASMSetFlags(fEFlags);
1591	return rc;
1592	}
1593
1594
1595	/**
1596	* Exits VMX root mode operation on the current CPU.
1597	*
1598	* @returns VBox status code.
1599	*/
1600	static int hmR0VmxLeaveRootMode(void)
1601	{
1602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603
1604	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1605	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1606
1607	/* If we're for some reason not in VMX root mode, then don't leave it. */
1608	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1609
1610	int rc;
1611	if (uHostCr4 & X86_CR4_VMXE)
1612	{
1613	/* Exit VMX root mode and clear the VMX bit in CR4. */
1614	VMXDisable();
1615	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1616	rc = VINF_SUCCESS;
1617	}
1618	else
1619	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1620
1621	/* Restore interrupts. */
1622	ASMSetFlags(fEFlags);
1623	return rc;
1624	}
1625
1626
1627	/**
1628	* Allocates and maps a physically contiguous page. The allocated page is
1629	* zero'd out (used by various VT-x structures).
1630	*
1631	* @returns IPRT status code.
1632	* @param pMemObj Pointer to the ring-0 memory object.
1633	* @param ppVirt Where to store the virtual address of the allocation.
1634	* @param pHCPhys Where to store the physical address of the allocation.
1635	*/
1636	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1637	{
1638	AssertPtr(pMemObj);
1639	AssertPtr(ppVirt);
1640	AssertPtr(pHCPhys);
1641	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1642	if (RT_FAILURE(rc))
1643	return rc;
1644	ppVirt = RTR0MemObjAddress(pMemObj);
1645	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1646	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1647	return VINF_SUCCESS;
1648	}
1649
1650
1651	/**
1652	* Frees and unmaps an allocated, physical page.
1653	*
1654	* @param pMemObj Pointer to the ring-0 memory object.
1655	* @param ppVirt Where to re-initialize the virtual address of allocation as
1656	* 0.
1657	* @param pHCPhys Where to re-initialize the physical address of the
1658	* allocation as 0.
1659	*/
1660	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1661	{
1662	AssertPtr(pMemObj);
1663	AssertPtr(ppVirt);
1664	AssertPtr(pHCPhys);
1665	/* NULL is valid, accepted and ignored by the free function below. */
1666	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1667	*pMemObj = NIL_RTR0MEMOBJ;
1668	*ppVirt = NULL;
1669	*pHCPhys = NIL_RTHCPHYS;
1670	}
1671
1672
1673	/**
1674	* Initializes a VMCS info. object.
1675	*
1676	* @param pVmcsInfo The VMCS info. object.
1677	*/
1678	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1679	{
1680	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1681
1682	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1683	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1684	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1685	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1686	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1687	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1688	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1689	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1690	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1691	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1692	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1693	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1694	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1695	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1696	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1697	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1698	}
1699
1700
1701	/**
1702	* Frees the VT-x structures for a VMCS info. object.
1703	*
1704	* @param pVM The cross context VM structure.
1705	* @param pVmcsInfo The VMCS info. object.
1706	*/
1707	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1708	{
1709	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1710
1711	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1712	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1713	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1714	#endif
1715
1716	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1717	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1718
1719	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1720	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1721	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1722
1723	hmR0VmxInitVmcsInfo(pVmcsInfo);
1724	}
1725
1726
1727	/**
1728	* Allocates the VT-x structures for a VMCS info. object.
1729	*
1730	* @returns VBox status code.
1731	* @param pVCpu The cross context virtual CPU structure.
1732	* @param pVmcsInfo The VMCS info. object.
1733	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1734	*/
1735	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1736	{
1737	PVM pVM = pVCpu->CTX_SUFF(pVM);
1738
1739	/* Allocate the guest VM control structure (VMCS). */
1740	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1741	if (RT_SUCCESS(rc))
1742	{
1743	if (!fIsNstGstVmcs)
1744	{
1745	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1746	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1747	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1748	#endif
1749	if (RT_SUCCESS(rc))
1750	{
1751	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1752	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1753	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1754	{
1755	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1756	NULL /* pR3Ptr /, NULL / pRCPtr */);
1757	}
1758	}
1759	}
1760	else
1761	{
1762	/* We don't yet support exposing VMCS shadowing to the guest. */
1763	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1764	Assert(!pVmcsInfo->pvShadowVmcs);
1765
1766	/* Get the allocated virtual-APIC page from CPUM. */
1767	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1768	{
1769	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1770	&pVmcsInfo->HCPhysVirtApic);
1771	Assert(pVmcsInfo->pbVirtApic);
1772	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1773	}
1774	}
1775
1776	if (RT_SUCCESS(rc))
1777	{
1778	/*
1779	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1780	* transparent accesses of specific MSRs.
1781	*
1782	* If the condition for enabling MSR bitmaps changes here, don't forget to
1783	* update HMIsMsrBitmapActive().
1784	*
1785	* We don't share MSR bitmaps between the guest and nested-guest as we then
1786	* don't need to care about carefully restoring the guest MSR bitmap.
1787	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1788	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1789	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1790	* we do that later while merging VMCS.
1791	*/
1792	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1793	{
1794	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1795	if ( RT_SUCCESS(rc)
1796	&& !fIsNstGstVmcs)
1797	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1798	}
1799
1800	if (RT_SUCCESS(rc))
1801	{
1802	/*
1803	* Allocate the VM-entry MSR-load area for the guest MSRs.
1804	*
1805	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1806	* the guest and nested-guest.
1807	*/
1808	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1809	&pVmcsInfo->HCPhysGuestMsrLoad);
1810	if (RT_SUCCESS(rc))
1811	{
1812	/*
1813	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1814	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1815	*/
1816	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1817	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1818	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1819
1820	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1821	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1822	&pVmcsInfo->HCPhysHostMsrLoad);
1823	}
1824	}
1825	}
1826	}
1827
1828	return rc;
1829	}
1830
1831
1832	/**
1833	* Free all VT-x structures for the VM.
1834	*
1835	* @returns IPRT status code.
1836	* @param pVM The cross context VM structure.
1837	*/
1838	static void hmR0VmxStructsFree(PVM pVM)
1839	{
1840	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1841	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1842	#endif
1843	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1844
1845	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1846	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1847	{
1848	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1849	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1850	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1851	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1852	}
1853	#endif
1854
1855	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1856	{
1857	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1858	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1859	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1860	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1861	if (pVM->cpum.ro.GuestFeatures.fVmx)
1862	{
1863	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1864	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1865	}
1866	#endif
1867	}
1868	}
1869
1870
1871	/**
1872	* Allocate all VT-x structures for the VM.
1873	*
1874	* @returns IPRT status code.
1875	* @param pVM The cross context VM structure.
1876	*/
1877	static int hmR0VmxStructsAlloc(PVM pVM)
1878	{
1879	/*
1880	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1881	* The VMCS size cannot be more than 4096 bytes.
1882	*
1883	* See Intel spec. Appendix A.1 "Basic VMX Information".
1884	*/
1885	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1886	if (cbVmcs <= X86_PAGE_4K_SIZE)
1887	{ /* likely */ }
1888	else
1889	{
1890	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1891	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1892	}
1893
1894	/*
1895	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1896	*/
1897	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1898	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1899	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1900	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1901	#endif
1902
1903	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1904	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1905	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1906
1907	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1908	{
1909	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1910	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1911	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1912	}
1913
1914	/*
1915	* Allocate per-VM VT-x structures.
1916	*/
1917	int rc = VINF_SUCCESS;
1918	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1919	/* Allocate crash-dump magic scratch page. */
1920	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1921	if (RT_FAILURE(rc))
1922	{
1923	hmR0VmxStructsFree(pVM);
1924	return rc;
1925	}
1926	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1927	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1928	#endif
1929
1930	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1931	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1932	{
1933	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1934	&pVM->hm.s.vmx.HCPhysApicAccess);
1935	if (RT_FAILURE(rc))
1936	{
1937	hmR0VmxStructsFree(pVM);
1938	return rc;
1939	}
1940	}
1941
1942	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1943	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1944	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1945	{
1946	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1947	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1948	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1949	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1950	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1951	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1952	{
1953	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1954	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1955	if (RT_SUCCESS(rc))
1956	{
1957	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1958	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1959	}
1960	}
1961	else
1962	rc = VERR_NO_MEMORY;
1963
1964	if (RT_FAILURE(rc))
1965	{
1966	hmR0VmxStructsFree(pVM);
1967	return rc;
1968	}
1969	}
1970	#endif
1971
1972	/*
1973	* Initialize per-VCPU VT-x structures.
1974	*/
1975	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1976	{
1977	/* Allocate the guest VMCS structures. */
1978	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1979	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1980	if (RT_SUCCESS(rc))
1981	{
1982	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1983	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1984	if (pVM->cpum.ro.GuestFeatures.fVmx)
1985	{
1986	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1987	if (RT_SUCCESS(rc))
1988	{ /* likely */ }
1989	else
1990	break;
1991	}
1992	#endif
1993	}
1994	else
1995	break;
1996	}
1997
1998	if (RT_FAILURE(rc))
1999	{
2000	hmR0VmxStructsFree(pVM);
2001	return rc;
2002	}
2003
2004	return VINF_SUCCESS;
2005	}
2006
2007	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2008	/**
2009	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2010	*
2011	* @returns @c true if the MSR is intercepted, @c false otherwise.
2012	* @param pvMsrBitmap The MSR bitmap.
2013	* @param offMsr The MSR byte offset.
2014	* @param iBit The bit offset from the byte offset.
2015	*/
2016	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2017	{
2018	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2019	Assert(pbMsrBitmap);
2020	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2021	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2022	}
2023	#endif
2024
2025	/**
2026	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2027	*
2028	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2029	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2030	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2031	* the read/write access of this MSR.
2032	*
2033	* @param pVCpu The cross context virtual CPU structure.
2034	* @param pVmcsInfo The VMCS info. object.
2035	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2036	* @param idMsr The MSR value.
2037	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2038	* include both a read -and- a write permission!
2039	*
2040	* @sa CPUMGetVmxMsrPermission.
2041	* @remarks Can be called with interrupts disabled.
2042	*/
2043	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2044	{
2045	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2046	Assert(pbMsrBitmap);
2047	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2048
2049	/*
2050	* MSR-bitmap Layout:
2051	* Byte index MSR range Interpreted as
2052	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2053	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2054	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2055	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2056	*
2057	* A bit corresponding to an MSR within the above range causes a VM-exit
2058	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2059	* the MSR range, it always cause a VM-exit.
2060	*
2061	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2062	*/
2063	uint16_t const offBitmapRead = 0;
2064	uint16_t const offBitmapWrite = 0x800;
2065	uint16_t offMsr;
2066	int32_t iBit;
2067	if (idMsr <= UINT32_C(0x00001fff))
2068	{
2069	offMsr = 0;
2070	iBit = idMsr;
2071	}
2072	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2073	{
2074	offMsr = 0x400;
2075	iBit = idMsr - UINT32_C(0xc0000000);
2076	}
2077	else
2078	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2079
2080	/*
2081	* Set the MSR read permission.
2082	*/
2083	uint16_t const offMsrRead = offBitmapRead + offMsr;
2084	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2085	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2086	{
2087	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2088	bool const fClear = !fIsNstGstVmcs ? true
2089	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2090	#else
2091	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2092	bool const fClear = true;
2093	#endif
2094	if (fClear)
2095	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2096	}
2097	else
2098	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2099
2100	/*
2101	* Set the MSR write permission.
2102	*/
2103	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2104	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2105	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2106	{
2107	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2108	bool const fClear = !fIsNstGstVmcs ? true
2109	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2110	#else
2111	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2112	bool const fClear = true;
2113	#endif
2114	if (fClear)
2115	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2116	}
2117	else
2118	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2119	}
2120
2121
2122	/**
2123	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2124	* area.
2125	*
2126	* @returns VBox status code.
2127	* @param pVCpu The cross context virtual CPU structure.
2128	* @param pVmcsInfo The VMCS info. object.
2129	* @param cMsrs The number of MSRs.
2130	*/
2131	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2132	{
2133	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2134	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2135	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2136	{
2137	/* Commit the MSR counts to the VMCS and update the cache. */
2138	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2139	{
2140	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2141	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2142	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2143	AssertRCReturn(rc, rc);
2144
2145	pVmcsInfo->cEntryMsrLoad = cMsrs;
2146	pVmcsInfo->cExitMsrStore = cMsrs;
2147	pVmcsInfo->cExitMsrLoad = cMsrs;
2148	}
2149	return VINF_SUCCESS;
2150	}
2151
2152	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2153	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2154	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2155	}
2156
2157
2158	/**
2159	* Adds a new (or updates the value of an existing) guest/host MSR
2160	* pair to be swapped during the world-switch as part of the
2161	* auto-load/store MSR area in the VMCS.
2162	*
2163	* @returns VBox status code.
2164	* @param pVCpu The cross context virtual CPU structure.
2165	* @param pVmxTransient The VMX-transient structure.
2166	* @param idMsr The MSR.
2167	* @param uGuestMsrValue Value of the guest MSR.
2168	* @param fSetReadWrite Whether to set the guest read/write access of this
2169	* MSR (thus not causing a VM-exit).
2170	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2171	* necessary.
2172	*/
2173	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2174	bool fSetReadWrite, bool fUpdateHostMsr)
2175	{
2176	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2177	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2178	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2179	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2180	uint32_t i;
2181
2182	/* Paranoia. */
2183	Assert(pGuestMsrLoad);
2184
2185	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2186
2187	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2188	for (i = 0; i < cMsrs; i++)
2189	{
2190	if (pGuestMsrLoad[i].u32Msr == idMsr)
2191	break;
2192	}
2193
2194	bool fAdded = false;
2195	if (i == cMsrs)
2196	{
2197	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2198	++cMsrs;
2199	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2200	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2201
2202	/* Set the guest to read/write this MSR without causing VM-exits. */
2203	if ( fSetReadWrite
2204	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2205	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2206
2207	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2208	fAdded = true;
2209	}
2210
2211	/* Update the MSR value for the newly added or already existing MSR. */
2212	pGuestMsrLoad[i].u32Msr = idMsr;
2213	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2214
2215	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2216	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2217	{
2218	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2219	pGuestMsrStore[i].u32Msr = idMsr;
2220	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2221	}
2222
2223	/* Update the corresponding slot in the host MSR area. */
2224	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2225	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2226	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2227	pHostMsr[i].u32Msr = idMsr;
2228
2229	/*
2230	* Only if the caller requests to update the host MSR value AND we've newly added the
2231	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2232	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2233	*
2234	* We do this for performance reasons since reading MSRs may be quite expensive.
2235	*/
2236	if (fAdded)
2237	{
2238	if (fUpdateHostMsr)
2239	{
2240	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2241	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2242	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2243	}
2244	else
2245	{
2246	/* Someone else can do the work. */
2247	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2248	}
2249	}
2250	return VINF_SUCCESS;
2251	}
2252
2253
2254	/**
2255	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2256	* auto-load/store MSR area in the VMCS.
2257	*
2258	* @returns VBox status code.
2259	* @param pVCpu The cross context virtual CPU structure.
2260	* @param pVmxTransient The VMX-transient structure.
2261	* @param idMsr The MSR.
2262	*/
2263	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2264	{
2265	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2266	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2267	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2268	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2269
2270	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2271
2272	for (uint32_t i = 0; i < cMsrs; i++)
2273	{
2274	/* Find the MSR. */
2275	if (pGuestMsrLoad[i].u32Msr == idMsr)
2276	{
2277	/*
2278	* If it's the last MSR, we only need to reduce the MSR count.
2279	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2280	*/
2281	if (i < cMsrs - 1)
2282	{
2283	/* Remove it from the VM-entry MSR-load area. */
2284	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2285	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2286
2287	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2288	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2289	{
2290	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2291	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2292	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2293	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2294	}
2295
2296	/* Remove it from the VM-exit MSR-load area. */
2297	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2298	Assert(pHostMsr[i].u32Msr == idMsr);
2299	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2300	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2301	}
2302
2303	/* Reduce the count to reflect the removed MSR and bail. */
2304	--cMsrs;
2305	break;
2306	}
2307	}
2308
2309	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2310	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2311	{
2312	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2313	AssertRCReturn(rc, rc);
2314
2315	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2316	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2317	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2318
2319	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2320	return VINF_SUCCESS;
2321	}
2322
2323	return VERR_NOT_FOUND;
2324	}
2325
2326
2327	/**
2328	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2329	*
2330	* @returns @c true if found, @c false otherwise.
2331	* @param pVmcsInfo The VMCS info. object.
2332	* @param idMsr The MSR to find.
2333	*/
2334	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2335	{
2336	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2337	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2338	Assert(pMsrs);
2339	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2340	for (uint32_t i = 0; i < cMsrs; i++)
2341	{
2342	if (pMsrs[i].u32Msr == idMsr)
2343	return true;
2344	}
2345	return false;
2346	}
2347
2348
2349	/**
2350	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2351	*
2352	* @param pVCpu The cross context virtual CPU structure.
2353	* @param pVmcsInfo The VMCS info. object.
2354	*
2355	* @remarks No-long-jump zone!!!
2356	*/
2357	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2358	{
2359	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2360
2361	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2362	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2363	Assert(pHostMsrLoad);
2364	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2365	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2366	for (uint32_t i = 0; i < cMsrs; i++)
2367	{
2368	/*
2369	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2370	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2371	*/
2372	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2373	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2374	else
2375	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2376	}
2377	}
2378
2379
2380	/**
2381	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2382	* perform lazy restoration of the host MSRs while leaving VT-x.
2383	*
2384	* @param pVCpu The cross context virtual CPU structure.
2385	*
2386	* @remarks No-long-jump zone!!!
2387	*/
2388	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2389	{
2390	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2391
2392	/*
2393	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2394	*/
2395	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2396	{
2397	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2398	#if HC_ARCH_BITS == 64
2399	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2400	{
2401	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2402	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2403	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2404	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2405	}
2406	#endif
2407	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2408	}
2409	}
2410
2411
2412	/**
2413	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2414	* lazily while leaving VT-x.
2415	*
2416	* @returns true if it does, false otherwise.
2417	* @param pVCpu The cross context virtual CPU structure.
2418	* @param idMsr The MSR to check.
2419	*/
2420	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2421	{
2422	NOREF(pVCpu);
2423	#if HC_ARCH_BITS == 64
2424	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2425	{
2426	switch (idMsr)
2427	{
2428	case MSR_K8_LSTAR:
2429	case MSR_K6_STAR:
2430	case MSR_K8_SF_MASK:
2431	case MSR_K8_KERNEL_GS_BASE:
2432	return true;
2433	}
2434	}
2435	#else
2436	RT_NOREF(pVCpu, idMsr);
2437	#endif
2438	return false;
2439	}
2440
2441
2442	/**
2443	* Loads a set of guests MSRs to allow read/passthru to the guest.
2444	*
2445	* The name of this function is slightly confusing. This function does NOT
2446	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2447	* common prefix for functions dealing with "lazy restoration" of the shared
2448	* MSRs.
2449	*
2450	* @param pVCpu The cross context virtual CPU structure.
2451	*
2452	* @remarks No-long-jump zone!!!
2453	*/
2454	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2455	{
2456	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2457	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2458
2459	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2460	#if HC_ARCH_BITS == 64
2461	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2462	{
2463	/*
2464	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2465	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2466	* we can skip a few MSR writes.
2467	*
2468	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2469	* guest MSR values in the guest-CPU context might be different to what's currently
2470	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2471	* CPU, see @bugref{8728}.
2472	*/
2473	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2474	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2475	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2476	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2477	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2478	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2479	{
2480	#ifdef VBOX_STRICT
2481	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2482	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2483	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2484	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2485	#endif
2486	}
2487	else
2488	{
2489	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2490	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2491	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2492	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2493	}
2494	}
2495	#endif
2496	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2497	}
2498
2499
2500	/**
2501	* Performs lazy restoration of the set of host MSRs if they were previously
2502	* loaded with guest MSR values.
2503	*
2504	* @param pVCpu The cross context virtual CPU structure.
2505	*
2506	* @remarks No-long-jump zone!!!
2507	* @remarks The guest MSRs should have been saved back into the guest-CPU
2508	* context by hmR0VmxImportGuestState()!!!
2509	*/
2510	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2511	{
2512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2513	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2514
2515	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2516	{
2517	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2518	#if HC_ARCH_BITS == 64
2519	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2520	{
2521	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2522	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2523	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2524	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2525	}
2526	#endif
2527	}
2528	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2529	}
2530
2531
2532	/**
2533	* Verifies that our cached values of the VMCS fields are all consistent with
2534	* what's actually present in the VMCS.
2535	*
2536	* @returns VBox status code.
2537	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2538	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2539	* VMCS content. HMCPU error-field is
2540	* updated, see VMX_VCI_XXX.
2541	* @param pVCpu The cross context virtual CPU structure.
2542	* @param pVmcsInfo The VMCS info. object.
2543	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2544	*/
2545	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2546	{
2547	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2548
2549	uint32_t u32Val;
2550	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2551	AssertRCReturn(rc, rc);
2552	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2553	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2554	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2555	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2556
2557	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2558	AssertRCReturn(rc, rc);
2559	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2560	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2561	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2562	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2563
2564	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2565	AssertRCReturn(rc, rc);
2566	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2567	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2568	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2569	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2570
2571	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2572	AssertRCReturn(rc, rc);
2573	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2574	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2575	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2576	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2577
2578	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2579	{
2580	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2581	AssertRCReturn(rc, rc);
2582	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2583	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2584	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2585	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2586	}
2587
2588	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2589	AssertRCReturn(rc, rc);
2590	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2591	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2592	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2593	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2594
2595	uint64_t u64Val;
2596	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2597	AssertRCReturn(rc, rc);
2598	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2599	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2600	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2601	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2602
2603	NOREF(pcszVmcs);
2604	return VINF_SUCCESS;
2605	}
2606
2607
2608	#ifdef VBOX_STRICT
2609	/**
2610	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2611	*
2612	* @param pVCpu The cross context virtual CPU structure.
2613	* @param pVmcsInfo The VMCS info. object.
2614	*/
2615	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2616	{
2617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2618
2619	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2620	{
2621	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2622	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2623	uint64_t uVmcsEferMsrVmcs;
2624	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2625	AssertRC(rc);
2626
2627	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2628	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2629	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2630	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2631	}
2632	}
2633
2634
2635	/**
2636	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2637	* VMCS are correct.
2638	*
2639	* @param pVCpu The cross context virtual CPU structure.
2640	* @param pVmcsInfo The VMCS info. object.
2641	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2642	*/
2643	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2644	{
2645	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2646
2647	/* Read the various MSR-area counts from the VMCS. */
2648	uint32_t cEntryLoadMsrs;
2649	uint32_t cExitStoreMsrs;
2650	uint32_t cExitLoadMsrs;
2651	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2652	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2653	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2654
2655	/* Verify all the MSR counts are the same. */
2656	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2657	Assert(cExitStoreMsrs == cExitLoadMsrs);
2658	uint32_t const cMsrs = cExitLoadMsrs;
2659
2660	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2661	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2662
2663	/* Verify the MSR counts are within the allocated page size. */
2664	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2665
2666	/* Verify the relevant contents of the MSR areas match. */
2667	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2668	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2669	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2670	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2671	for (uint32_t i = 0; i < cMsrs; i++)
2672	{
2673	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2674	if (fSeparateExitMsrStorePage)
2675	{
2676	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2677	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2678	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2679	}
2680
2681	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2682	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2683	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2684
2685	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2686	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2687	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2688	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2689
2690	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2691	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2692	if (fIsEferMsr)
2693	{
2694	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2695	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2696	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2697	}
2698
2699	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2700	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2701	{
2702	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2703	if (fIsEferMsr)
2704	{
2705	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2706	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2707	}
2708	else
2709	{
2710	if (!fIsNstGstVmcs)
2711	{
2712	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2713	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2714	}
2715	else
2716	{
2717	/*
2718	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2719	* execute a nested-guest with MSR passthrough.
2720	*
2721	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2722	* allow passthrough too.
2723	*/
2724	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2725	Assert(pvMsrBitmapNstGst);
2726	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2727	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2728	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2729	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2730	}
2731	}
2732	}
2733
2734	/* Move to the next MSR. */
2735	pHostMsrLoad++;
2736	pGuestMsrLoad++;
2737	pGuestMsrStore++;
2738	}
2739	}
2740	#endif /* VBOX_STRICT */
2741
2742
2743	/**
2744	* Flushes the TLB using EPT.
2745	*
2746	* @returns VBox status code.
2747	* @param pVCpu The cross context virtual CPU structure of the calling
2748	* EMT. Can be NULL depending on @a enmTlbFlush.
2749	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2750	* enmTlbFlush.
2751	* @param enmTlbFlush Type of flush.
2752	*
2753	* @remarks Caller is responsible for making sure this function is called only
2754	* when NestedPaging is supported and providing @a enmTlbFlush that is
2755	* supported by the CPU.
2756	* @remarks Can be called with interrupts disabled.
2757	*/
2758	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2759	{
2760	uint64_t au64Descriptor[2];
2761	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2762	au64Descriptor[0] = 0;
2763	else
2764	{
2765	Assert(pVCpu);
2766	Assert(pVmcsInfo);
2767	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2768	}
2769	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2770
2771	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2772	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2773
2774	if ( RT_SUCCESS(rc)
2775	&& pVCpu)
2776	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2777	}
2778
2779
2780	/**
2781	* Flushes the TLB using VPID.
2782	*
2783	* @returns VBox status code.
2784	* @param pVCpu The cross context virtual CPU structure of the calling
2785	* EMT. Can be NULL depending on @a enmTlbFlush.
2786	* @param enmTlbFlush Type of flush.
2787	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2788	* on @a enmTlbFlush).
2789	*
2790	* @remarks Can be called with interrupts disabled.
2791	*/
2792	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2793	{
2794	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2795
2796	uint64_t au64Descriptor[2];
2797	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2798	{
2799	au64Descriptor[0] = 0;
2800	au64Descriptor[1] = 0;
2801	}
2802	else
2803	{
2804	AssertPtr(pVCpu);
2805	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2806	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2807	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2808	au64Descriptor[1] = GCPtr;
2809	}
2810
2811	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2812	AssertMsg(rc == VINF_SUCCESS,
2813	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2814
2815	if ( RT_SUCCESS(rc)
2816	&& pVCpu)
2817	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2818	NOREF(rc);
2819	}
2820
2821
2822	/**
2823	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2824	* otherwise there is nothing really to invalidate.
2825	*
2826	* @returns VBox status code.
2827	* @param pVCpu The cross context virtual CPU structure.
2828	* @param GCVirt Guest virtual address of the page to invalidate.
2829	*/
2830	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2831	{
2832	AssertPtr(pVCpu);
2833	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2834
2835	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2836	{
2837	/*
2838	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2839	* the EPT case. See @bugref{6043} and @bugref{6177}.
2840	*
2841	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2842	* as this function maybe called in a loop with individual addresses.
2843	*/
2844	PVM pVM = pVCpu->CTX_SUFF(pVM);
2845	if (pVM->hm.s.vmx.fVpid)
2846	{
2847	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2848
2849	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2850	/*
2851	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2852	* where executing INVVPID outside 64-bit mode does not flush translations of
2853	* 64-bit linear addresses, see @bugref{6208#c72}.
2854	*/
2855	if (RT_HI_U32(GCVirt))
2856	fVpidFlush = false;
2857	#endif
2858
2859	if (fVpidFlush)
2860	{
2861	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2862	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2863	}
2864	else
2865	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2866	}
2867	else if (pVM->hm.s.fNestedPaging)
2868	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2869	}
2870
2871	return VINF_SUCCESS;
2872	}
2873
2874
2875	/**
2876	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2877	* case where neither EPT nor VPID is supported by the CPU.
2878	*
2879	* @param pHostCpu The HM physical-CPU structure.
2880	* @param pVCpu The cross context virtual CPU structure.
2881	*
2882	* @remarks Called with interrupts disabled.
2883	*/
2884	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2885	{
2886	AssertPtr(pVCpu);
2887	AssertPtr(pHostCpu);
2888
2889	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2890
2891	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2892	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2893	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2894	pVCpu->hm.s.fForceTLBFlush = false;
2895	return;
2896	}
2897
2898
2899	/**
2900	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2901	*
2902	* @param pHostCpu The HM physical-CPU structure.
2903	* @param pVCpu The cross context virtual CPU structure.
2904	* @param pVmcsInfo The VMCS info. object.
2905	*
2906	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2907	* nomenclature. The reason is, to avoid confusion in compare statements
2908	* since the host-CPU copies are named "ASID".
2909	*
2910	* @remarks Called with interrupts disabled.
2911	*/
2912	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2913	{
2914	#ifdef VBOX_WITH_STATISTICS
2915	bool fTlbFlushed = false;
2916	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2917	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2918	if (!fTlbFlushed) \
2919	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2920	} while (0)
2921	#else
2922	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2923	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2924	#endif
2925
2926	AssertPtr(pVCpu);
2927	AssertPtr(pHostCpu);
2928	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2929
2930	PVM pVM = pVCpu->CTX_SUFF(pVM);
2931	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2932	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2933	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2934
2935	/*
2936	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2937	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2938	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2939	* cannot reuse the current ASID anymore.
2940	*/
2941	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2942	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2943	{
2944	++pHostCpu->uCurrentAsid;
2945	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2946	{
2947	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2948	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2949	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2950	}
2951
2952	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2953	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2954	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2955
2956	/*
2957	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2958	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2959	*/
2960	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2961	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2962	HMVMX_SET_TAGGED_TLB_FLUSHED();
2963	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2964	}
2965	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2966	{
2967	/*
2968	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2969	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2970	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2971	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2972	* mappings, see @bugref{6568}.
2973	*
2974	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2975	*/
2976	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2977	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2978	HMVMX_SET_TAGGED_TLB_FLUSHED();
2979	}
2980
2981	pVCpu->hm.s.fForceTLBFlush = false;
2982	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2983
2984	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2985	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2986	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2987	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2988	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2989	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2990	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2991	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2992	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2993
2994	/* Update VMCS with the VPID. */
2995	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2996	AssertRC(rc);
2997
2998	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2999	}
3000
3001
3002	/**
3003	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3004	*
3005	* @param pHostCpu The HM physical-CPU structure.
3006	* @param pVCpu The cross context virtual CPU structure.
3007	* @param pVmcsInfo The VMCS info. object.
3008	*
3009	* @remarks Called with interrupts disabled.
3010	*/
3011	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3012	{
3013	AssertPtr(pVCpu);
3014	AssertPtr(pHostCpu);
3015	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3016	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3017	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3018
3019	/*
3020	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3021	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3022	*/
3023	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3024	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3025	{
3026	pVCpu->hm.s.fForceTLBFlush = true;
3027	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3028	}
3029
3030	/* Check for explicit TLB flushes. */
3031	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3032	{
3033	pVCpu->hm.s.fForceTLBFlush = true;
3034	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3035	}
3036
3037	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3038	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3039
3040	if (pVCpu->hm.s.fForceTLBFlush)
3041	{
3042	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3043	pVCpu->hm.s.fForceTLBFlush = false;
3044	}
3045	}
3046
3047
3048	/**
3049	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3050	*
3051	* @param pHostCpu The HM physical-CPU structure.
3052	* @param pVCpu The cross context virtual CPU structure.
3053	*
3054	* @remarks Called with interrupts disabled.
3055	*/
3056	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3057	{
3058	AssertPtr(pVCpu);
3059	AssertPtr(pHostCpu);
3060	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3061	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3062	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3063
3064	/*
3065	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3066	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3067	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3068	* cannot reuse the current ASID anymore.
3069	*/
3070	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3071	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3072	{
3073	pVCpu->hm.s.fForceTLBFlush = true;
3074	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3075	}
3076
3077	/* Check for explicit TLB flushes. */
3078	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3079	{
3080	/*
3081	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3082	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3083	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3084	* include fExplicitFlush's too) - an obscure corner case.
3085	*/
3086	pVCpu->hm.s.fForceTLBFlush = true;
3087	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3088	}
3089
3090	PVM pVM = pVCpu->CTX_SUFF(pVM);
3091	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3092	if (pVCpu->hm.s.fForceTLBFlush)
3093	{
3094	++pHostCpu->uCurrentAsid;
3095	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3096	{
3097	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3098	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3099	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3100	}
3101
3102	pVCpu->hm.s.fForceTLBFlush = false;
3103	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3104	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3105	if (pHostCpu->fFlushAsidBeforeUse)
3106	{
3107	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3108	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3109	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3110	{
3111	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3112	pHostCpu->fFlushAsidBeforeUse = false;
3113	}
3114	else
3115	{
3116	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3117	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3118	}
3119	}
3120	}
3121
3122	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3123	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3124	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3125	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3126	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3127	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3128	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3129
3130	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3131	AssertRC(rc);
3132	}
3133
3134
3135	/**
3136	* Flushes the guest TLB entry based on CPU capabilities.
3137	*
3138	* @param pHostCpu The HM physical-CPU structure.
3139	* @param pVCpu The cross context virtual CPU structure.
3140	* @param pVmcsInfo The VMCS info. object.
3141	*
3142	* @remarks Called with interrupts disabled.
3143	*/
3144	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3145	{
3146	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3147	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3148	#endif
3149	PVM pVM = pVCpu->CTX_SUFF(pVM);
3150	switch (pVM->hm.s.vmx.enmTlbFlushType)
3151	{
3152	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3153	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3154	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3155	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3156	default:
3157	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3158	break;
3159	}
3160	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3161	}
3162
3163
3164	/**
3165	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3166	* TLB entries from the host TLB before VM-entry.
3167	*
3168	* @returns VBox status code.
3169	* @param pVM The cross context VM structure.
3170	*/
3171	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3172	{
3173	/*
3174	* Determine optimal flush type for nested paging.
3175	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3176	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3177	*/
3178	if (pVM->hm.s.fNestedPaging)
3179	{
3180	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3181	{
3182	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3183	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3184	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3185	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3186	else
3187	{
3188	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3189	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3190	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3191	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3192	}
3193
3194	/* Make sure the write-back cacheable memory type for EPT is supported. */
3195	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3196	{
3197	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3198	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3199	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3200	}
3201
3202	/* EPT requires a page-walk length of 4. */
3203	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3204	{
3205	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3206	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3207	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3208	}
3209	}
3210	else
3211	{
3212	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3213	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3214	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3215	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3216	}
3217	}
3218
3219	/*
3220	* Determine optimal flush type for VPID.
3221	*/
3222	if (pVM->hm.s.vmx.fVpid)
3223	{
3224	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3225	{
3226	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3227	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3228	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3229	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3230	else
3231	{
3232	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3233	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3234	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3235	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3236	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3237	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3238	pVM->hm.s.vmx.fVpid = false;
3239	}
3240	}
3241	else
3242	{
3243	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3244	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3245	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3246	pVM->hm.s.vmx.fVpid = false;
3247	}
3248	}
3249
3250	/*
3251	* Setup the handler for flushing tagged-TLBs.
3252	*/
3253	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3254	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3255	else if (pVM->hm.s.fNestedPaging)
3256	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3257	else if (pVM->hm.s.vmx.fVpid)
3258	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3259	else
3260	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3261	return VINF_SUCCESS;
3262	}
3263
3264
3265	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3266	/**
3267	* Sets up the shadow VMCS fields arrays.
3268	*
3269	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3270	* executing the guest.
3271	*
3272	* @returns VBox status code.
3273	* @param pVM The cross context VM structure.
3274	*/
3275	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3276	{
3277	/*
3278	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3279	* when the host does not support it.
3280	*/
3281	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3282	if ( !fGstVmwriteAll
3283	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3284	{ /* likely. */ }
3285	else
3286	{
3287	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3288	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3289	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3290	}
3291
3292	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3293	uint32_t cRwFields = 0;
3294	uint32_t cRoFields = 0;
3295	for (uint32_t i = 0; i < cVmcsFields; i++)
3296	{
3297	VMXVMCSFIELD VmcsField;
3298	VmcsField.u = g_aVmcsFields[i];
3299
3300	/*
3301	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3302	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3303	* in the shadow VMCS fields array as they would be redundant.
3304	*
3305	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3306	* we must not include it in the shadow VMCS fields array. Guests attempting to
3307	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3308	* the required behavior.
3309	*/
3310	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3311	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3312	{
3313	/*
3314	* Read-only fields are placed in a separate array so that while syncing shadow
3315	* VMCS fields later (which is more performance critical) we can avoid branches.
3316	*
3317	* However, if the guest can write to all fields (including read-only fields),
3318	* we treat it a as read/write field. Otherwise, writing to these fields would
3319	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3320	*/
3321	if ( fGstVmwriteAll
3322	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3323	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3324	else
3325	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3326	}
3327	}
3328
3329	/* Update the counts. */
3330	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3331	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3332	return VINF_SUCCESS;
3333	}
3334
3335
3336	/**
3337	* Sets up the VMREAD and VMWRITE bitmaps.
3338	*
3339	* @param pVM The cross context VM structure.
3340	*/
3341	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3342	{
3343	/*
3344	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3345	*/
3346	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3347	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3348	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3349	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3350	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3351
3352	/*
3353	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3354	* VMREAD and VMWRITE bitmaps.
3355	*/
3356	{
3357	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3358	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3359	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3360	{
3361	uint32_t const uVmcsField = paShadowVmcsFields[i];
3362	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3363	Assert(uVmcsField >> 3 < cbBitmap);
3364	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3365	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3366	}
3367	}
3368
3369	/*
3370	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3371	* if the host supports VMWRITE to all supported VMCS fields.
3372	*/
3373	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3374	{
3375	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3376	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3377	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3378	{
3379	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3380	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3381	Assert(uVmcsField >> 3 < cbBitmap);
3382	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3383	}
3384	}
3385	}
3386	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3387
3388
3389	/**
3390	* Sets up the virtual-APIC page address for the VMCS.
3391	*
3392	* @returns VBox status code.
3393	* @param pVCpu The cross context virtual CPU structure.
3394	* @param pVmcsInfo The VMCS info. object.
3395	*/
3396	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3397	{
3398	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3399	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3400	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3401	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3402	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3403	}
3404
3405
3406	/**
3407	* Sets up the MSR-bitmap address for the VMCS.
3408	*
3409	* @returns VBox status code.
3410	* @param pVCpu The cross context virtual CPU structure.
3411	* @param pVmcsInfo The VMCS info. object.
3412	*/
3413	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3414	{
3415	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3416	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3417	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3418	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3419	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3420	}
3421
3422
3423	/**
3424	* Sets up the APIC-access page address for the VMCS.
3425	*
3426	* @returns VBox status code.
3427	* @param pVCpu The cross context virtual CPU structure.
3428	*/
3429	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3430	{
3431	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3432	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3433	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3434	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3435	}
3436
3437
3438	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3439	/**
3440	* Sets up the VMREAD bitmap address for the VMCS.
3441	*
3442	* @returns VBox status code.
3443	* @param pVCpu The cross context virtual CPU structure.
3444	*/
3445	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3446	{
3447	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3448	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3449	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3450	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3451	}
3452
3453
3454	/**
3455	* Sets up the VMWRITE bitmap address for the VMCS.
3456	*
3457	* @returns VBox status code.
3458	* @param pVCpu The cross context virtual CPU structure.
3459	*/
3460	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3461	{
3462	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3463	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3464	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3465	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3466	}
3467	#endif
3468
3469
3470	/**
3471	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3472	* in the VMCS.
3473	*
3474	* @returns VBox status code.
3475	* @param pVCpu The cross context virtual CPU structure.
3476	* @param pVmcsInfo The VMCS info. object.
3477	*/
3478	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3479	{
3480	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3481
3482	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3483	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3484	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3485
3486	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3487	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3488	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3489
3490	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3491	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3492	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3493
3494	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3495	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3496	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3497	AssertRCReturn(rc, rc);
3498	return VINF_SUCCESS;
3499	}
3500
3501
3502	/**
3503	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3504	*
3505	* @param pVCpu The cross context virtual CPU structure.
3506	* @param pVmcsInfo The VMCS info. object.
3507	*/
3508	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3509	{
3510	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3511
3512	/*
3513	* The guest can access the following MSRs (read, write) without causing
3514	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3515	*/
3516	PVM pVM = pVCpu->CTX_SUFF(pVM);
3517	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3518	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3519	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3520	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3521	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3522
3523	/*
3524	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3525	* associated with then. We never need to intercept access (writes need to be
3526	* executed without causing a VM-exit, reads will #GP fault anyway).
3527	*
3528	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3529	* read/write them. We swap the the guest/host MSR value using the
3530	* auto-load/store MSR area.
3531	*/
3532	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3533	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3534	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3535	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3536	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3537	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3538
3539	#if HC_ARCH_BITS == 64
3540	/*
3541	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3542	* required for 64-bit guests.
3543	*/
3544	if (pVM->hm.s.fAllow64BitGuests)
3545	{
3546	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3547	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3548	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3549	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3550	}
3551	#endif
3552
3553	/*
3554	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3555	*/
3556	#ifdef VBOX_STRICT
3557	Assert(pVmcsInfo->pvMsrBitmap);
3558	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3559	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3560	#endif
3561	}
3562
3563
3564	/**
3565	* Sets up pin-based VM-execution controls in the VMCS.
3566	*
3567	* @returns VBox status code.
3568	* @param pVCpu The cross context virtual CPU structure.
3569	* @param pVmcsInfo The VMCS info. object.
3570	*/
3571	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3572	{
3573	PVM pVM = pVCpu->CTX_SUFF(pVM);
3574	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3575	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3576
3577	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3578	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3579
3580	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3581	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3582
3583	/* Enable the VMX-preemption timer. */
3584	if (pVM->hm.s.vmx.fUsePreemptTimer)
3585	{
3586	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3587	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3588	}
3589
3590	#if 0
3591	/* Enable posted-interrupt processing. */
3592	if (pVM->hm.s.fPostedIntrs)
3593	{
3594	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3595	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3596	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3597	}
3598	#endif
3599
3600	if ((fVal & fZap) != fVal)
3601	{
3602	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3603	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3604	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3605	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3606	}
3607
3608	/* Commit it to the VMCS and update our cache. */
3609	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3610	AssertRCReturn(rc, rc);
3611	pVmcsInfo->u32PinCtls = fVal;
3612
3613	return VINF_SUCCESS;
3614	}
3615
3616
3617	/**
3618	* Sets up secondary processor-based VM-execution controls in the VMCS.
3619	*
3620	* @returns VBox status code.
3621	* @param pVCpu The cross context virtual CPU structure.
3622	* @param pVmcsInfo The VMCS info. object.
3623	*/
3624	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3625	{
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3628	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3629
3630	/* WBINVD causes a VM-exit. */
3631	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3632	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3633
3634	/* Enable EPT (aka nested-paging). */
3635	if (pVM->hm.s.fNestedPaging)
3636	fVal \|= VMX_PROC_CTLS2_EPT;
3637
3638	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3639	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3640	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3641	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3642	fVal \|= VMX_PROC_CTLS2_INVPCID;
3643
3644	/* Enable VPID. */
3645	if (pVM->hm.s.vmx.fVpid)
3646	fVal \|= VMX_PROC_CTLS2_VPID;
3647
3648	/* Enable unrestricted guest execution. */
3649	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3650	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3651
3652	#if 0
3653	if (pVM->hm.s.fVirtApicRegs)
3654	{
3655	/* Enable APIC-register virtualization. */
3656	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3657	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3658
3659	/* Enable virtual-interrupt delivery. */
3660	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3661	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3662	}
3663	#endif
3664
3665	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3666	where the TPR shadow resides. */
3667	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3668	* done dynamically. */
3669	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3670	{
3671	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3672	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3673	AssertRCReturn(rc, rc);
3674	}
3675
3676	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3677	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3678	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3679	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3680	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3681
3682	/* Enable Pause-Loop exiting. */
3683	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3684	&& pVM->hm.s.vmx.cPleGapTicks
3685	&& pVM->hm.s.vmx.cPleWindowTicks)
3686	{
3687	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3688
3689	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3690	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3691	AssertRCReturn(rc, rc);
3692	}
3693
3694	if ((fVal & fZap) != fVal)
3695	{
3696	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3697	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3698	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3699	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3700	}
3701
3702	/* Commit it to the VMCS and update our cache. */
3703	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3704	AssertRCReturn(rc, rc);
3705	pVmcsInfo->u32ProcCtls2 = fVal;
3706
3707	return VINF_SUCCESS;
3708	}
3709
3710
3711	/**
3712	* Sets up processor-based VM-execution controls in the VMCS.
3713	*
3714	* @returns VBox status code.
3715	* @param pVCpu The cross context virtual CPU structure.
3716	* @param pVmcsInfo The VMCS info. object.
3717	*/
3718	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3719	{
3720	PVM pVM = pVCpu->CTX_SUFF(pVM);
3721
3722	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3723	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3724
3725	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3726	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3727	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3728	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3729	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3730	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3731	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3732
3733	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3734	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3735	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3736	{
3737	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3738	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3739	}
3740
3741	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3742	if (!pVM->hm.s.fNestedPaging)
3743	{
3744	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3745	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3746	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3747	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3748	}
3749
3750	/* Use TPR shadowing if supported by the CPU. */
3751	if ( PDMHasApic(pVM)
3752	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3753	{
3754	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3755	/* CR8 writes cause a VM-exit based on TPR threshold. */
3756	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3757	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3758	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3759	AssertRCReturn(rc, rc);
3760	}
3761	else
3762	{
3763	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3764	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3765	if (pVM->hm.s.fAllow64BitGuests)
3766	{
3767	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3768	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3769	}
3770	}
3771
3772	/* Use MSR-bitmaps if supported by the CPU. */
3773	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3774	{
3775	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3776	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3777	AssertRCReturn(rc, rc);
3778	}
3779
3780	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3781	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3782	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3783
3784	if ((fVal & fZap) != fVal)
3785	{
3786	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3787	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3788	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3789	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3790	}
3791
3792	/* Commit it to the VMCS and update our cache. */
3793	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3794	AssertRCReturn(rc, rc);
3795	pVmcsInfo->u32ProcCtls = fVal;
3796
3797	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3798	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3799	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3800
3801	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3802	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3803	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3804
3805	/* Sanity check, should not really happen. */
3806	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3807	{ /* likely */ }
3808	else
3809	{
3810	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3811	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3812	}
3813
3814	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3815	return VINF_SUCCESS;
3816	}
3817
3818
3819	/**
3820	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3821	* Processor-based VM-execution) control fields in the VMCS.
3822	*
3823	* @returns VBox status code.
3824	* @param pVCpu The cross context virtual CPU structure.
3825	* @param pVmcsInfo The VMCS info. object.
3826	*/
3827	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3828	{
3829	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3830	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3831	{
3832	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3833	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3834	if (RT_SUCCESS(rc))
3835	{ /* likely */ }
3836	else
3837	{
3838	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3839	return rc;
3840	}
3841	}
3842	#endif
3843
3844	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3845	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3846	if (RT_SUCCESS(rc))
3847	{
3848	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3849	if (RT_SUCCESS(rc))
3850	{
3851	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3852	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3853	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3854	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3855	if (RT_SUCCESS(rc))
3856	{
3857	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3858	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3859	return VINF_SUCCESS;
3860	}
3861	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3862	}
3863	else
3864	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3865	}
3866	else
3867	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3868	return rc;
3869	}
3870
3871
3872	/**
3873	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3874	*
3875	* We shall setup those exception intercepts that don't change during the
3876	* lifetime of the VM here. The rest are done dynamically while loading the
3877	* guest state.
3878	*
3879	* @returns VBox status code.
3880	* @param pVCpu The cross context virtual CPU structure.
3881	* @param pVmcsInfo The VMCS info. object.
3882	*/
3883	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3884	{
3885	/*
3886	* The following exceptions are always intercepted:
3887	*
3888	* #AC - To prevent the guest from hanging the CPU.
3889	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3890	* recursive #DBs can cause a CPU hang.
3891	* #PF - To sync our shadow page tables when nested-paging is not used.
3892	*/
3893	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3894	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3895	\| RT_BIT(X86_XCPT_DB)
3896	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3897
3898	/* Commit it to the VMCS. */
3899	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3900	AssertRCReturn(rc, rc);
3901
3902	/* Update our cache of the exception bitmap. */
3903	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3904	return VINF_SUCCESS;
3905	}
3906
3907
3908	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3909	/**
3910	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3911	*
3912	* @returns VBox status code.
3913	* @param pVCpu The cross context virtual CPU structure.
3914	* @param pVmcsInfo The VMCS info. object.
3915	*/
3916	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3917	{
3918	PVM pVM = pVCpu->CTX_SUFF(pVM);
3919	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3920	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3921	if (RT_SUCCESS(rc))
3922	{
3923	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3924	if (RT_SUCCESS(rc))
3925	{
3926	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3927	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3928	if (RT_SUCCESS(rc))
3929	{
3930	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3931	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3932	if (RT_SUCCESS(rc))
3933	return VINF_SUCCESS;
3934
3935	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3936	}
3937	else
3938	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3939	}
3940	else
3941	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3942	}
3943	else
3944	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3945
3946	return rc;
3947	}
3948	#endif
3949
3950
3951	/**
3952	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3953	* VMX.
3954	*
3955	* @returns VBox status code.
3956	* @param pVCpu The cross context virtual CPU structure.
3957	* @param pVmcsInfo The VMCS info. object.
3958	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3959	*/
3960	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3961	{
3962	Assert(pVmcsInfo->pvVmcs);
3963	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3964
3965	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3966	PVM pVM = pVCpu->CTX_SUFF(pVM);
3967	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3968	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3969
3970	LogFlowFunc(("\n"));
3971
3972	/*
3973	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3974	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3975	*/
3976	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3977	if (RT_SUCCESS(rc))
3978	{
3979	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3980	if (RT_SUCCESS(rc))
3981	{
3982	if (!fIsNstGstVmcs)
3983	{
3984	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3985	if (RT_SUCCESS(rc))
3986	{
3987	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3988	if (RT_SUCCESS(rc))
3989	{
3990	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3991	if (RT_SUCCESS(rc))
3992	{
3993	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3994	if (RT_SUCCESS(rc))
3995	{
3996	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3997	/*
3998	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
3999	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4000	* making it fit for use when VMCS shadowing is later enabled.
4001	*/
4002	if (pVmcsInfo->pvShadowVmcs)
4003	{
4004	VMXVMCSREVID VmcsRevId;
4005	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4006	VmcsRevId.n.fIsShadowVmcs = 1;
4007	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4008	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4009	if (RT_SUCCESS(rc))
4010	{ /* likely */ }
4011	else
4012	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4013	}
4014	#endif
4015	}
4016	else
4017	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4018	}
4019	else
4020	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4021	}
4022	else
4023	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4024	}
4025	else
4026	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4027	}
4028	else
4029	{
4030	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4031	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4032	if (RT_SUCCESS(rc))
4033	{ /* likely */ }
4034	else
4035	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4036	#else
4037	AssertFailed();
4038	#endif
4039	}
4040	}
4041	else
4042	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4043	}
4044	else
4045	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4046
4047	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4048	if (RT_SUCCESS(rc))
4049	{
4050	rc = hmR0VmxClearVmcs(pVmcsInfo);
4051	if (RT_SUCCESS(rc))
4052	{ /* likely */ }
4053	else
4054	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4055	}
4056
4057	/*
4058	* Update the last-error record both for failures and success, so we
4059	* can propagate the status code back to ring-3 for diagnostics.
4060	*/
4061	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4062	NOREF(pszVmcs);
4063	return rc;
4064	}
4065
4066
4067	/**
4068	* Does global VT-x initialization (called during module initialization).
4069	*
4070	* @returns VBox status code.
4071	*/
4072	VMMR0DECL(int) VMXR0GlobalInit(void)
4073	{
4074	#ifdef HMVMX_USE_FUNCTION_TABLE
4075	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4076	# ifdef VBOX_STRICT
4077	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4078	Assert(g_apfnVMExitHandlers[i]);
4079	# endif
4080	#endif
4081	return VINF_SUCCESS;
4082	}
4083
4084
4085	/**
4086	* Does global VT-x termination (called during module termination).
4087	*/
4088	VMMR0DECL(void) VMXR0GlobalTerm()
4089	{
4090	/* Nothing to do currently. */
4091	}
4092
4093
4094	/**
4095	* Sets up and activates VT-x on the current CPU.
4096	*
4097	* @returns VBox status code.
4098	* @param pHostCpu The HM physical-CPU structure.
4099	* @param pVM The cross context VM structure. Can be
4100	* NULL after a host resume operation.
4101	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4102	* fEnabledByHost is @c true).
4103	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4104	* @a fEnabledByHost is @c true).
4105	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4106	* enable VT-x on the host.
4107	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4108	*/
4109	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4110	PCSUPHWVIRTMSRS pHwvirtMsrs)
4111	{
4112	AssertPtr(pHostCpu);
4113	AssertPtr(pHwvirtMsrs);
4114	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4115
4116	/* Enable VT-x if it's not already enabled by the host. */
4117	if (!fEnabledByHost)
4118	{
4119	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4120	if (RT_FAILURE(rc))
4121	return rc;
4122	}
4123
4124	/*
4125	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4126	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4127	* invalidated when flushing by VPID.
4128	*/
4129	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4130	{
4131	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4132	pHostCpu->fFlushAsidBeforeUse = false;
4133	}
4134	else
4135	pHostCpu->fFlushAsidBeforeUse = true;
4136
4137	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4138	++pHostCpu->cTlbFlushes;
4139
4140	return VINF_SUCCESS;
4141	}
4142
4143
4144	/**
4145	* Deactivates VT-x on the current CPU.
4146	*
4147	* @returns VBox status code.
4148	* @param pvCpuPage Pointer to the VMXON region.
4149	* @param HCPhysCpuPage Physical address of the VMXON region.
4150	*
4151	* @remarks This function should never be called when SUPR0EnableVTx() or
4152	* similar was used to enable VT-x on the host.
4153	*/
4154	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4155	{
4156	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4157
4158	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4159	return hmR0VmxLeaveRootMode();
4160	}
4161
4162
4163	/**
4164	* Does per-VM VT-x initialization.
4165	*
4166	* @returns VBox status code.
4167	* @param pVM The cross context VM structure.
4168	*/
4169	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4170	{
4171	AssertPtr(pVM);
4172	LogFlowFunc(("pVM=%p\n", pVM));
4173
4174	int rc = hmR0VmxStructsAlloc(pVM);
4175	if (RT_FAILURE(rc))
4176	{
4177	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4178	return rc;
4179	}
4180
4181	return VINF_SUCCESS;
4182	}
4183
4184
4185	/**
4186	* Does per-VM VT-x termination.
4187	*
4188	* @returns VBox status code.
4189	* @param pVM The cross context VM structure.
4190	*/
4191	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4192	{
4193	AssertPtr(pVM);
4194	LogFlowFunc(("pVM=%p\n", pVM));
4195
4196	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4197	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4198	{
4199	Assert(pVM->hm.s.vmx.pvScratch);
4200	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4201	}
4202	#endif
4203	hmR0VmxStructsFree(pVM);
4204	return VINF_SUCCESS;
4205	}
4206
4207
4208	/**
4209	* Sets up the VM for execution using hardware-assisted VMX.
4210	* This function is only called once per-VM during initialization.
4211	*
4212	* @returns VBox status code.
4213	* @param pVM The cross context VM structure.
4214	*/
4215	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4216	{
4217	AssertPtr(pVM);
4218	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4219
4220	LogFlowFunc(("pVM=%p\n", pVM));
4221
4222	/*
4223	* At least verify if VMX is enabled, since we can't check if we're in
4224	* VMX root mode or not without causing a #GP.
4225	*/
4226	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4227	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4228	{ /* likely */ }
4229	else
4230	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4231
4232	/*
4233	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4234	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4235	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4236	*/
4237	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4238	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4239	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4240	{
4241	LogRelFunc(("Invalid real-on-v86 state.\n"));
4242	return VERR_INTERNAL_ERROR;
4243	}
4244
4245	/* Initialize these always, see hmR3InitFinalizeR0().*/
4246	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4247	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4248
4249	/* Setup the tagged-TLB flush handlers. */
4250	int rc = hmR0VmxSetupTaggedTlb(pVM);
4251	if (RT_FAILURE(rc))
4252	{
4253	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4254	return rc;
4255	}
4256
4257	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4258	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4259	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4260	{
4261	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4262	if (RT_SUCCESS(rc))
4263	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4264	else
4265	{
4266	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4267	return rc;
4268	}
4269	}
4270	#endif
4271
4272	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4273	{
4274	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4275	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4276
4277	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4278	if (RT_SUCCESS(rc))
4279	{
4280	#if HC_ARCH_BITS == 32
4281	hmR0VmxInitVmcsReadCache(pVCpu);
4282	#endif
4283	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4284	if (pVM->cpum.ro.GuestFeatures.fVmx)
4285	{
4286	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4287	if (RT_SUCCESS(rc))
4288	{ /* likely */ }
4289	else
4290	{
4291	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4292	return rc;
4293	}
4294	}
4295	#endif
4296	}
4297	else
4298	{
4299	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4300	return rc;
4301	}
4302	}
4303
4304	return VINF_SUCCESS;
4305	}
4306
4307
4308	#if HC_ARCH_BITS == 32
4309	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4310	/**
4311	* Check if guest state allows safe use of 32-bit switcher again.
4312	*
4313	* Segment bases and protected mode structures must be 32-bit addressable
4314	* because the 32-bit switcher will ignore high dword when writing these VMCS
4315	* fields. See @bugref{8432} for details.
4316	*
4317	* @returns true if safe, false if must continue to use the 64-bit switcher.
4318	* @param pCtx Pointer to the guest-CPU context.
4319	*
4320	* @remarks No-long-jump zone!!!
4321	*/
4322	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4323	{
4324	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4325	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4326	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4327	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4328	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4329	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4330	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4331	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4332	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4333	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4334
4335	/* All good, bases are 32-bit. */
4336	return true;
4337	}
4338	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4339
4340	# ifdef VBOX_STRICT
4341	static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4342	{
4343	switch (idxField)
4344	{
4345	case VMX_VMCS_GUEST_RIP:
4346	case VMX_VMCS_GUEST_RSP:
4347	case VMX_VMCS_GUEST_SYSENTER_EIP:
4348	case VMX_VMCS_GUEST_SYSENTER_ESP:
4349	case VMX_VMCS_GUEST_GDTR_BASE:
4350	case VMX_VMCS_GUEST_IDTR_BASE:
4351	case VMX_VMCS_GUEST_CS_BASE:
4352	case VMX_VMCS_GUEST_DS_BASE:
4353	case VMX_VMCS_GUEST_ES_BASE:
4354	case VMX_VMCS_GUEST_FS_BASE:
4355	case VMX_VMCS_GUEST_GS_BASE:
4356	case VMX_VMCS_GUEST_SS_BASE:
4357	case VMX_VMCS_GUEST_LDTR_BASE:
4358	case VMX_VMCS_GUEST_TR_BASE:
4359	case VMX_VMCS_GUEST_CR3:
4360	return true;
4361	}
4362	return false;
4363	}
4364
4365	static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4366	{
4367	switch (idxField)
4368	{
4369	/* Read-only fields. */
4370	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4371	return true;
4372	}
4373	/* Remaining readable fields should also be writable. */
4374	return hmR0VmxIsValidWriteFieldInCache(idxField);
4375	}
4376	# endif /* VBOX_STRICT */
4377
4378
4379	/**
4380	* Executes the specified handler in 64-bit mode.
4381	*
4382	* @returns VBox status code (no informational status codes).
4383	* @param pVCpu The cross context virtual CPU structure.
4384	* @param enmOp The operation to perform.
4385	* @param cParams Number of parameters.
4386	* @param paParam Array of 32-bit parameters.
4387	*/
4388	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4389	{
4390	AssertPtr(pVCpu);
4391	PVM pVM = pVCpu->CTX_SUFF(pVM);
4392	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4393	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4394	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4395	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4396
4397	#ifdef VBOX_STRICT
4398	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4399	Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4400
4401	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4402	Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4403	#endif
4404
4405	/* Disable interrupts. */
4406	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4407
4408	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4409	RTCPUID idHostCpu = RTMpCpuId();
4410	CPUMR0SetLApic(pVCpu, idHostCpu);
4411	#endif
4412
4413	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4414
4415	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4416	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4417
4418	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4419	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4420	hmR0VmxClearVmcs(pVmcsInfo);
4421
4422	/* Leave VMX root mode and disable VMX. */
4423	VMXDisable();
4424	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4425
4426	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4427	CPUMSetHyperEIP(pVCpu, enmOp);
4428	for (int i = (int)cParams - 1; i >= 0; i--)
4429	CPUMPushHyper(pVCpu, paParam[i]);
4430
4431	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4432
4433	/* Call the switcher. */
4434	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4435	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4436
4437	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4438	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4439
4440	/* Re-enter VMX root mode. */
4441	int rc2 = VMXEnable(HCPhysCpuPage);
4442	if (RT_FAILURE(rc2))
4443	{
4444	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4445	ASMSetFlags(fOldEFlags);
4446	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4447	return rc2;
4448	}
4449
4450	/* Restore the VMCS as the current VMCS. */
4451	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4452	AssertRC(rc2);
4453	Assert(!(ASMGetFlags() & X86_EFL_IF));
4454	ASMSetFlags(fOldEFlags);
4455	return rc;
4456	}
4457
4458
4459	/**
4460	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4461	* supporting 64-bit guests.
4462	*
4463	* @returns VBox status code.
4464	* @param fResume Whether to VMLAUNCH or VMRESUME.
4465	* @param pCtx Pointer to the guest-CPU context.
4466	* @param pCache Pointer to the VMCS batch cache.
4467	* @param pVM The cross context VM structure.
4468	* @param pVCpu The cross context virtual CPU structure.
4469	*/
4470	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4471	{
4472	NOREF(fResume);
4473
4474	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4475	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4476	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4477
4478	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4479	pCache->uPos = 1;
4480	pCache->interPD = PGMGetInterPaeCR3(pVM);
4481	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4482	#endif
4483
4484	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4485	pCache->TestIn.HCPhysCpuPage = 0;
4486	pCache->TestIn.HCPhysVmcs = 0;
4487	pCache->TestIn.pCache = 0;
4488	pCache->TestOut.HCPhysVmcs = 0;
4489	pCache->TestOut.pCache = 0;
4490	pCache->TestOut.pCtx = 0;
4491	pCache->TestOut.eflags = 0;
4492	#else
4493	NOREF(pCache);
4494	#endif
4495
4496	uint32_t aParam[10];
4497	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4498	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4499	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4500	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4501	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4502	aParam[5] = 0;
4503	aParam[6] = VM_RC_ADDR(pVM, pVM);
4504	aParam[7] = 0;
4505	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4506	aParam[9] = 0;
4507
4508	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4509	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4510	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4511	#endif
4512	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4513
4514	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4515	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4516	Assert(pCtx->dr[4] == 10);
4517	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4518	#endif
4519
4520	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4521	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4522	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4523	pVmcsInfo->HCPhysVmcs));
4524	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4525	pCache->TestOut.HCPhysVmcs));
4526	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4527	pCache->TestOut.pCache));
4528	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4529	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4530	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4531	pCache->TestOut.pCtx));
4532	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4533	#endif
4534	NOREF(pCtx);
4535	return rc;
4536	}
4537	#endif
4538
4539
4540	/**
4541	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4542	* the VMCS.
4543	*
4544	* @returns VBox status code.
4545	*/
4546	static int hmR0VmxExportHostControlRegs(void)
4547	{
4548	RTCCUINTREG uReg = ASMGetCR0();
4549	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4550	AssertRCReturn(rc, rc);
4551
4552	uReg = ASMGetCR3();
4553	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4554	AssertRCReturn(rc, rc);
4555
4556	uReg = ASMGetCR4();
4557	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4558	AssertRCReturn(rc, rc);
4559	return rc;
4560	}
4561
4562
4563	/**
4564	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4565	* the host-state area in the VMCS.
4566	*
4567	* @returns VBox status code.
4568	* @param pVCpu The cross context virtual CPU structure.
4569	*/
4570	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4571	{
4572	#if HC_ARCH_BITS == 64
4573	/**
4574	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4575	* requirements. See hmR0VmxExportHostSegmentRegs().
4576	*/
4577	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4578	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4579	{ \
4580	bool fValidSelector = true; \
4581	if ((selValue) & X86_SEL_LDT) \
4582	{ \
4583	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4584	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4585	} \
4586	if (fValidSelector) \
4587	{ \
4588	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4589	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4590	} \
4591	(selValue) = 0; \
4592	}
4593
4594	/*
4595	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4596	* will be messed up. We should -not- save the messed up state without restoring
4597	* the original host-state, see @bugref{7240}.
4598	*
4599	* This apparently can happen (most likely the FPU changes), deal with it rather than
4600	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4601	*/
4602	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4603	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4604	{
4605	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4606	pVCpu->idCpu));
4607	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4608	}
4609	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4610	#else
4611	RT_NOREF(pVCpu);
4612	#endif
4613
4614	/*
4615	* Host DS, ES, FS and GS segment registers.
4616	*/
4617	#if HC_ARCH_BITS == 64
4618	RTSEL uSelDS = ASMGetDS();
4619	RTSEL uSelES = ASMGetES();
4620	RTSEL uSelFS = ASMGetFS();
4621	RTSEL uSelGS = ASMGetGS();
4622	#else
4623	RTSEL uSelDS = 0;
4624	RTSEL uSelES = 0;
4625	RTSEL uSelFS = 0;
4626	RTSEL uSelGS = 0;
4627	#endif
4628
4629	/*
4630	* Host CS and SS segment registers.
4631	*/
4632	RTSEL uSelCS = ASMGetCS();
4633	RTSEL uSelSS = ASMGetSS();
4634
4635	/*
4636	* Host TR segment register.
4637	*/
4638	RTSEL uSelTR = ASMGetTR();
4639
4640	#if HC_ARCH_BITS == 64
4641	/*
4642	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4643	* gain VM-entry and restore them before we get preempted.
4644	*
4645	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4646	*/
4647	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4648	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4649	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4650	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4651	# undef VMXLOCAL_ADJUST_HOST_SEG
4652	#endif
4653
4654	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4655	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4656	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4657	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4658	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4659	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4660	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4661	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4662	Assert(uSelCS);
4663	Assert(uSelTR);
4664
4665	/* Write these host selector fields into the host-state area in the VMCS. */
4666	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4667	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4668	#if HC_ARCH_BITS == 64
4669	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4670	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4671	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4672	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4673	#else
4674	NOREF(uSelDS);
4675	NOREF(uSelES);
4676	NOREF(uSelFS);
4677	NOREF(uSelGS);
4678	#endif
4679	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4680	AssertRCReturn(rc, rc);
4681
4682	/*
4683	* Host GDTR and IDTR.
4684	*/
4685	RTGDTR Gdtr;
4686	RTIDTR Idtr;
4687	RT_ZERO(Gdtr);
4688	RT_ZERO(Idtr);
4689	ASMGetGDTR(&Gdtr);
4690	ASMGetIDTR(&Idtr);
4691	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4692	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4693	AssertRCReturn(rc, rc);
4694
4695	#if HC_ARCH_BITS == 64
4696	/*
4697	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4698	* them to the maximum limit (0xffff) on every VM-exit.
4699	*/
4700	if (Gdtr.cbGdt != 0xffff)
4701	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4702
4703	/*
4704	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4705	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4706	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4707	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4708	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4709	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4710	* at 0xffff on hosts where we are sure it won't cause trouble.
4711	*/
4712	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4713	if (Idtr.cbIdt < 0x0fff)
4714	# else
4715	if (Idtr.cbIdt != 0xffff)
4716	# endif
4717	{
4718	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4719	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4720	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4721	}
4722	#endif
4723
4724	/*
4725	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4726	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4727	* RPL should be too in most cases.
4728	*/
4729	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4730	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4731
4732	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4733	#if HC_ARCH_BITS == 64
4734	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4735
4736	/*
4737	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4738	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4739	* restoration if the host has something else. Task switching is not supported in 64-bit
4740	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4741	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4742	*
4743	* [1] See Intel spec. 3.5 "System Descriptor Types".
4744	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4745	*/
4746	PVM pVM = pVCpu->CTX_SUFF(pVM);
4747	Assert(pDesc->System.u4Type == 11);
4748	if ( pDesc->System.u16LimitLow != 0x67
4749	\|\| pDesc->System.u4LimitHigh)
4750	{
4751	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4752	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4753	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4754	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4755	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4756	}
4757
4758	/*
4759	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4760	*/
4761	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4762	{
4763	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4764	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4765	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4766	{
4767	/* The GDT is read-only but the writable GDT is available. */
4768	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4769	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4770	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4771	AssertRCReturn(rc, rc);
4772	}
4773	}
4774	#else
4775	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4776	#endif
4777	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4778	AssertRCReturn(rc, rc);
4779
4780	/*
4781	* Host FS base and GS base.
4782	*/
4783	#if HC_ARCH_BITS == 64
4784	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4785	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4786	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4787	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4788	AssertRCReturn(rc, rc);
4789
4790	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4791	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4792	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4793	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4794	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4795	#endif
4796	return VINF_SUCCESS;
4797	}
4798
4799
4800	/**
4801	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4802	* host-state area of the VMCS.
4803	*
4804	* These MSRs will be automatically restored on the host after every successful
4805	* VM-exit.
4806	*
4807	* @returns VBox status code.
4808	* @param pVCpu The cross context virtual CPU structure.
4809	*
4810	* @remarks No-long-jump zone!!!
4811	*/
4812	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4813	{
4814	AssertPtr(pVCpu);
4815
4816	/*
4817	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4818	* rather than swapping them on every VM-entry.
4819	*/
4820	hmR0VmxLazySaveHostMsrs(pVCpu);
4821
4822	/*
4823	* Host Sysenter MSRs.
4824	*/
4825	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4826	#if HC_ARCH_BITS == 32
4827	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4828	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4829	#else
4830	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4831	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4832	#endif
4833	AssertRCReturn(rc, rc);
4834
4835	/*
4836	* Host EFER MSR.
4837	*
4838	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4839	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4840	*/
4841	PVM pVM = pVCpu->CTX_SUFF(pVM);
4842	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4843	{
4844	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4845	AssertRCReturn(rc, rc);
4846	}
4847
4848	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4849	* hmR0VmxExportGuestEntryExitCtls(). */
4850
4851	return VINF_SUCCESS;
4852	}
4853
4854
4855	/**
4856	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4857	*
4858	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4859	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4860	*
4861	* @returns true if we need to load guest EFER, false otherwise.
4862	* @param pVCpu The cross context virtual CPU structure.
4863	*
4864	* @remarks Requires EFER, CR4.
4865	* @remarks No-long-jump zone!!!
4866	*/
4867	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4868	{
4869	#ifdef HMVMX_ALWAYS_SWAP_EFER
4870	RT_NOREF(pVCpu);
4871	return true;
4872	#else
4873	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4874	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4875	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4876	if (CPUMIsGuestInLongModeEx(pCtx))
4877	return false;
4878	#endif
4879
4880	PVM pVM = pVCpu->CTX_SUFF(pVM);
4881	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4882	uint64_t const u64GuestEfer = pCtx->msrEFER;
4883
4884	/*
4885	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4886	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4887	*/
4888	if ( CPUMIsGuestInLongModeEx(pCtx)
4889	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4890	return true;
4891
4892	/*
4893	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4894	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4895	*
4896	* See Intel spec. 4.5 "IA-32e Paging".
4897	* See Intel spec. 4.1.1 "Three Paging Modes".
4898	*
4899	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4900	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4901	*/
4902	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4903	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4904	if ( (pCtx->cr4 & X86_CR4_PAE)
4905	&& (pCtx->cr0 & X86_CR0_PG)
4906	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4907	{
4908	/* Assert that host is NX capable. */
4909	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4910	return true;
4911	}
4912
4913	return false;
4914	#endif
4915	}
4916
4917	/**
4918	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4919	* VMCS.
4920	*
4921	* This is typically required when the guest changes paging mode.
4922	*
4923	* @returns VBox status code.
4924	* @param pVCpu The cross context virtual CPU structure.
4925	* @param pVmxTransient The VMX-transient structure.
4926	*
4927	* @remarks Requires EFER.
4928	* @remarks No-long-jump zone!!!
4929	*/
4930	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4931	{
4932	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4933	{
4934	PVM pVM = pVCpu->CTX_SUFF(pVM);
4935	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4936
4937	/*
4938	* VM-entry controls.
4939	*/
4940	{
4941	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4942	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4943
4944	/*
4945	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4946	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4947	*
4948	* For nested-guests, this is a mandatory VM-entry control. It's also
4949	* required because we do not want to leak host bits to the nested-guest.
4950	*/
4951	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4952
4953	/*
4954	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4955	*
4956	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4957	* required to get the nested-guest working with hardware-assisted VMX execution.
4958	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4959	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4960	* here rather than while merging the guest VMCS controls.
4961	*/
4962	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4963	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4964	else
4965	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4966
4967	/*
4968	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4969	*
4970	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4971	* regardless of whether the nested-guest VMCS specifies it because we are free to
4972	* load whatever MSRs we require and we do not need to modify the guest visible copy
4973	* of the VM-entry MSR load area.
4974	*/
4975	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4976	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4977	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4978	else
4979	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4980
4981	/*
4982	* The following should -not- be set (since we're not in SMM mode):
4983	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4984	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4985	*/
4986
4987	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4988	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4989
4990	if ((fVal & fZap) == fVal)
4991	{ /* likely */ }
4992	else
4993	{
4994	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4995	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4996	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4997	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4998	}
4999
5000	/* Commit it to the VMCS. */
5001	if (pVmcsInfo->u32EntryCtls != fVal)
5002	{
5003	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
5004	AssertRCReturn(rc, rc);
5005	pVmcsInfo->u32EntryCtls = fVal;
5006	}
5007	}
5008
5009	/*
5010	* VM-exit controls.
5011	*/
5012	{
5013	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
5014	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
5015
5016	/*
5017	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
5018	* supported the 1-setting of this bit.
5019	*
5020	* For nested-guests, we set the "save debug controls" as the converse
5021	* "load debug controls" is mandatory for nested-guests anyway.
5022	*/
5023	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
5024
5025	/*
5026	* Set the host long mode active (EFER.LMA) bit (which Intel calls
5027	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
5028	* host EFER.LMA and EFER.LME bit to this value. See assertion in
5029	* hmR0VmxExportHostMsrs().
5030	*
5031	* For nested-guests, we always set this bit as we do not support 32-bit
5032	* hosts.
5033	*/
5034	#if HC_ARCH_BITS == 64
5035	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5036	#else
5037	Assert(!pVmxTransient->fIsNestedGuest);
5038	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5039	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5040	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5041	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5042	{
5043	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5044	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5045	}
5046	else
5047	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5048	#endif
5049
5050	/*
5051	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
5052	*
5053	* For nested-guests, we should use the "save IA32_EFER" control if we also
5054	* used the "load IA32_EFER" control while exporting VM-entry controls.
5055	*/
5056	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5057	&& hmR0VmxShouldSwapEferMsr(pVCpu))
5058	{
5059	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
5060	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
5061	}
5062
5063	/*
5064	* Enable saving of the VMX-preemption timer value on VM-exit.
5065	* For nested-guests, currently not exposed/used.
5066	*/
5067	if ( pVM->hm.s.vmx.fUsePreemptTimer
5068	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5069	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5070
5071	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5072	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5073
5074	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5075	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
5076	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5077
5078	if ((fVal & fZap) == fVal)
5079	{ /* likely */ }
5080	else
5081	{
5082	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5083	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5084	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5085	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5086	}
5087
5088	/* Commit it to the VMCS. */
5089	if (pVmcsInfo->u32ExitCtls != fVal)
5090	{
5091	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5092	AssertRCReturn(rc, rc);
5093	pVmcsInfo->u32ExitCtls = fVal;
5094	}
5095	}
5096
5097	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5098	}
5099	return VINF_SUCCESS;
5100	}
5101
5102
5103	/**
5104	* Sets the TPR threshold in the VMCS.
5105	*
5106	* @returns VBox status code.
5107	* @param pVCpu The cross context virtual CPU structure.
5108	* @param pVmcsInfo The VMCS info. object.
5109	* @param u32TprThreshold The TPR threshold (task-priority class only).
5110	*/
5111	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5112	{
5113	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5114	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5115	RT_NOREF2(pVCpu, pVmcsInfo);
5116	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5117	}
5118
5119
5120	/**
5121	* Exports the guest APIC TPR state into the VMCS.
5122	*
5123	* @returns VBox status code.
5124	* @param pVCpu The cross context virtual CPU structure.
5125	* @param pVmxTransient The VMX-transient structure.
5126	*
5127	* @remarks No-long-jump zone!!!
5128	*/
5129	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5130	{
5131	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5132	{
5133	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5134
5135	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5136	if (!pVmxTransient->fIsNestedGuest)
5137	{
5138	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5139	&& APICIsEnabled(pVCpu))
5140	{
5141	/*
5142	* Setup TPR shadowing.
5143	*/
5144	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5145	{
5146	bool fPendingIntr = false;
5147	uint8_t u8Tpr = 0;
5148	uint8_t u8PendingIntr = 0;
5149	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5150	AssertRCReturn(rc, rc);
5151
5152	/*
5153	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5154	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5155	* priority of the pending interrupt so we can deliver the interrupt. If there
5156	* are no interrupts pending, set threshold to 0 to not cause any
5157	* TPR-below-threshold VM-exits.
5158	*/
5159	Assert(pVmcsInfo->pbVirtApic);
5160	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5161	uint32_t u32TprThreshold = 0;
5162	if (fPendingIntr)
5163	{
5164	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5165	(which is the Task-Priority Class). */
5166	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5167	const uint8_t u8TprPriority = u8Tpr >> 4;
5168	if (u8PendingPriority <= u8TprPriority)
5169	u32TprThreshold = u8PendingPriority;
5170	}
5171
5172	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5173	AssertRCReturn(rc, rc);
5174	}
5175	}
5176	}
5177	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5178	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5179	}
5180	return VINF_SUCCESS;
5181	}
5182
5183
5184	/**
5185	* Gets the guest interruptibility-state.
5186	*
5187	* @returns Guest's interruptibility-state.
5188	* @param pVCpu The cross context virtual CPU structure.
5189	* @param pVmxTransient The VMX-transient structure.
5190	*
5191	* @remarks No-long-jump zone!!!
5192	*/
5193	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5194	{
5195	/*
5196	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5197	*/
5198	uint32_t fIntrState = 0;
5199	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5200	{
5201	/* If inhibition is active, RIP and RFLAGS should've been updated
5202	(i.e. read previously from the VMCS or from ring-3). */
5203	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5204	#ifdef VBOX_STRICT
5205	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5206	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5207	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5208	#endif
5209	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5210	{
5211	if (pCtx->eflags.Bits.u1IF)
5212	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5213	else
5214	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5215	}
5216	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5217	{
5218	/*
5219	* We can clear the inhibit force flag as even if we go back to the recompiler
5220	* without executing guest code in VT-x, the flag's condition to be cleared is
5221	* met and thus the cleared state is correct.
5222	*/
5223	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5224	}
5225	}
5226
5227	/*
5228	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5229	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5230	* setting this would block host-NMIs and IRET will not clear the blocking.
5231	*
5232	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5233	*
5234	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5235	*/
5236	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5237	&& CPUMIsGuestNmiBlocking(pVCpu))
5238	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5239
5240	return fIntrState;
5241	}
5242
5243
5244	/**
5245	* Exports the exception intercepts required for guest execution in the VMCS.
5246	*
5247	* @returns VBox status code.
5248	* @param pVCpu The cross context virtual CPU structure.
5249	* @param pVmxTransient The VMX-transient structure.
5250	*
5251	* @remarks No-long-jump zone!!!
5252	*/
5253	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5254	{
5255	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5256	{
5257	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5258	if ( !pVmxTransient->fIsNestedGuest
5259	&& pVCpu->hm.s.fGIMTrapXcptUD)
5260	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5261	else
5262	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5263
5264	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5265	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5266	}
5267	return VINF_SUCCESS;
5268	}
5269
5270
5271	/**
5272	* Exports the guest's RIP into the guest-state area in the VMCS.
5273	*
5274	* @returns VBox status code.
5275	* @param pVCpu The cross context virtual CPU structure.
5276	*
5277	* @remarks No-long-jump zone!!!
5278	*/
5279	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5280	{
5281	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5282	{
5283	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5284
5285	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5286	AssertRCReturn(rc, rc);
5287
5288	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5289	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5290	}
5291	return VINF_SUCCESS;
5292	}
5293
5294
5295	/**
5296	* Exports the guest's RSP into the guest-state area in the VMCS.
5297	*
5298	* @returns VBox status code.
5299	* @param pVCpu The cross context virtual CPU structure.
5300	*
5301	* @remarks No-long-jump zone!!!
5302	*/
5303	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5304	{
5305	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5306	{
5307	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5308
5309	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5310	AssertRCReturn(rc, rc);
5311
5312	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5313	}
5314	return VINF_SUCCESS;
5315	}
5316
5317
5318	/**
5319	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5320	*
5321	* @returns VBox status code.
5322	* @param pVCpu The cross context virtual CPU structure.
5323	* @param pVmxTransient The VMX-transient structure.
5324	*
5325	* @remarks No-long-jump zone!!!
5326	*/
5327	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5328	{
5329	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5330	{
5331	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5332
5333	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5334	Let us assert it as such and use 32-bit VMWRITE. */
5335	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5336	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5337	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5338	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5339
5340	/*
5341	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5342	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5343	* can run the real-mode guest code under Virtual 8086 mode.
5344	*/
5345	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5346	if (pVmcsInfo->RealMode.fRealOnV86Active)
5347	{
5348	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5349	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5350	Assert(!pVmxTransient->fIsNestedGuest);
5351	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5352	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5353	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5354	}
5355
5356	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5357	AssertRCReturn(rc, rc);
5358
5359	/*
5360	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5361	*
5362	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5363	* through the hypervisor debugger using EFLAGS.TF.
5364	*/
5365	if ( !pVmxTransient->fIsNestedGuest
5366	&& !pVCpu->hm.s.fSingleInstruction
5367	&& fEFlags.Bits.u1TF)
5368	{
5369	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5370	* premature trips to ring-3 esp since IEM does not yet handle it. */
5371	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5372	AssertRCReturn(rc, rc);
5373	}
5374	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5375	* nested-guest VMCS. */
5376
5377	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5378	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5379	}
5380	return VINF_SUCCESS;
5381	}
5382
5383
5384	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5385	/**
5386	* Copies the nested-guest VMCS to the shadow VMCS.
5387	*
5388	* @returns VBox status code.
5389	* @param pVCpu The cross context virtual CPU structure.
5390	* @param pVmcsInfo The VMCS info. object.
5391	*
5392	* @remarks No-long-jump zone!!!
5393	*/
5394	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5395	{
5396	PVM pVM = pVCpu->CTX_SUFF(pVM);
5397	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5398
5399	/*
5400	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5401	* current VMCS, as we may try saving guest lazy MSRs.
5402	*
5403	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5404	* calling the import VMCS code which is currently performing the guest MSR reads
5405	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5406	* and the rest of the VMX leave session machinery.
5407	*/
5408	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5409
5410	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5411	if (RT_SUCCESS(rc))
5412	{
5413	/*
5414	* Copy all guest read/write VMCS fields.
5415	*
5416	* We don't check for VMWRITE failures here for performance reasons and
5417	* because they are not expected to fail, barring irrecoverable conditions
5418	* like hardware errors.
5419	*/
5420	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5421	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5422	{
5423	uint64_t u64Val;
5424	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5425	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5426	VMXWriteVmcs64(uVmcsField, u64Val);
5427	}
5428
5429	/*
5430	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5431	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5432	*/
5433	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5434	{
5435	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5436	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5437	{
5438	uint64_t u64Val;
5439	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5440	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5441	VMXWriteVmcs64(uVmcsField, u64Val);
5442	}
5443	}
5444
5445	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5446	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5447	}
5448
5449	ASMSetFlags(fEFlags);
5450	return rc;
5451	}
5452
5453
5454	/**
5455	* Copies the shadow VMCS to the nested-guest VMCS.
5456	*
5457	* @returns VBox status code.
5458	* @param pVCpu The cross context virtual CPU structure.
5459	* @param pVmcsInfo The VMCS info. object.
5460	*
5461	* @remarks Called with interrupts disabled.
5462	*/
5463	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5464	{
5465	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5466	PVM pVM = pVCpu->CTX_SUFF(pVM);
5467	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5468
5469	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5470	if (RT_SUCCESS(rc))
5471	{
5472	/*
5473	* Copy guest read/write fields from the shadow VMCS.
5474	* Guest read-only fields cannot be modified, so no need to copy them.
5475	*
5476	* We don't check for VMREAD failures here for performance reasons and
5477	* because they are not expected to fail, barring irrecoverable conditions
5478	* like hardware errors.
5479	*/
5480	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5481	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5482	{
5483	uint64_t u64Val;
5484	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5485	VMXReadVmcs64(uVmcsField, &u64Val);
5486	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5487	}
5488
5489	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5490	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5491	}
5492	return rc;
5493	}
5494
5495
5496	/**
5497	* Enables VMCS shadowing for the given VMCS info. object.
5498	*
5499	* @param pVCpu The cross context virtual CPU structure.
5500	* @param pVmcsInfo The VMCS info. object.
5501	*
5502	* @remarks No-long-jump zone!!!
5503	*/
5504	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5505	{
5506	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5507
5508	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5509	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5510	{
5511	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5512	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5513	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5514	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5515	AssertRC(rc);
5516
5517	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5518	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5519	Log4Func(("Enabled\n"));
5520	}
5521	}
5522
5523
5524	/**
5525	* Disables VMCS shadowing for the given VMCS info. object.
5526	*
5527	* @param pVCpu The cross context virtual CPU structure.
5528	* @param pVmcsInfo The VMCS info. object.
5529	*
5530	* @remarks No-long-jump zone!!!
5531	*/
5532	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5533	{
5534	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5535
5536	/*
5537	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5538	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5539	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5540	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5541	*
5542	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5543	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5544	*/
5545	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5546	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5547	{
5548	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5549	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5550	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5551	AssertRC(rc);
5552
5553	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5554	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5555	Log4Func(("Disabled\n"));
5556	}
5557	}
5558	#endif
5559
5560
5561	/**
5562	* Exports the guest hardware-virtualization state.
5563	*
5564	* @returns VBox status code.
5565	* @param pVCpu The cross context virtual CPU structure.
5566	* @param pVmxTransient The VMX-transient structure.
5567	*
5568	* @remarks No-long-jump zone!!!
5569	*/
5570	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5571	{
5572	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5573	{
5574	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5575	/*
5576	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5577	* VMCS shadowing.
5578	*/
5579	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5580	{
5581	/*
5582	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5583	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5584	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5585	*
5586	* We check for VMX root mode here in case the guest executes VMXOFF without
5587	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5588	* not clear the current VMCS pointer.
5589	*/
5590	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5591	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5592	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5593	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5594	{
5595	/* Paranoia. */
5596	Assert(!pVmxTransient->fIsNestedGuest);
5597
5598	/*
5599	* For performance reasons, also check if the guest hypervisor's current VMCS
5600	* was newly loaded or modified before copying it to the shadow VMCS.
5601	*/
5602	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5603	{
5604	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5605	AssertRCReturn(rc, rc);
5606	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5607	}
5608	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5609	}
5610	else
5611	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5612	}
5613	#else
5614	NOREF(pVmxTransient);
5615	#endif
5616	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5617	}
5618	return VINF_SUCCESS;
5619	}
5620
5621
5622	/**
5623	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5624	*
5625	* The guest FPU state is always pre-loaded hence we don't need to bother about
5626	* sharing FPU related CR0 bits between the guest and host.
5627	*
5628	* @returns VBox status code.
5629	* @param pVCpu The cross context virtual CPU structure.
5630	* @param pVmxTransient The VMX-transient structure.
5631	*
5632	* @remarks No-long-jump zone!!!
5633	*/
5634	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5635	{
5636	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5637	{
5638	PVM pVM = pVCpu->CTX_SUFF(pVM);
5639	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5640
5641	/*
5642	* Figure out fixed CR0 bits in VMX operation.
5643	*/
5644	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5645	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5646	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5647	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5648	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5649	else
5650	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5651
5652	if (!pVmxTransient->fIsNestedGuest)
5653	{
5654	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5655	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5656	uint64_t const u64ShadowCr0 = u64GuestCr0;
5657	Assert(!RT_HI_U32(u64GuestCr0));
5658
5659	/*
5660	* Setup VT-x's view of the guest CR0.
5661	*/
5662	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5663	if (pVM->hm.s.fNestedPaging)
5664	{
5665	if (CPUMIsGuestPagingEnabled(pVCpu))
5666	{
5667	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5668	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5669	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5670	}
5671	else
5672	{
5673	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5674	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5675	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5676	}
5677
5678	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5679	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5680	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5681	}
5682	else
5683	{
5684	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5685	u64GuestCr0 \|= X86_CR0_WP;
5686	}
5687
5688	/*
5689	* Guest FPU bits.
5690	*
5691	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5692	* using CR0.TS.
5693	*
5694	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5695	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5696	*/
5697	u64GuestCr0 \|= X86_CR0_NE;
5698
5699	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5700	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5701
5702	/*
5703	* Update exception intercepts.
5704	*/
5705	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5706	if (pVmcsInfo->RealMode.fRealOnV86Active)
5707	{
5708	Assert(PDMVmmDevHeapIsEnabled(pVM));
5709	Assert(pVM->hm.s.vmx.pRealModeTSS);
5710	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5711	}
5712	else
5713	{
5714	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5715	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5716	if (fInterceptMF)
5717	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5718	}
5719
5720	/* Additional intercepts for debugging, define these yourself explicitly. */
5721	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5722	uXcptBitmap \|= 0
5723	\| RT_BIT(X86_XCPT_BP)
5724	\| RT_BIT(X86_XCPT_DE)
5725	\| RT_BIT(X86_XCPT_NM)
5726	\| RT_BIT(X86_XCPT_TS)
5727	\| RT_BIT(X86_XCPT_UD)
5728	\| RT_BIT(X86_XCPT_NP)
5729	\| RT_BIT(X86_XCPT_SS)
5730	\| RT_BIT(X86_XCPT_GP)
5731	\| RT_BIT(X86_XCPT_PF)
5732	\| RT_BIT(X86_XCPT_MF)
5733	;
5734	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5735	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5736	#endif
5737	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5738	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5739	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5740
5741	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5742	u64GuestCr0 \|= fSetCr0;
5743	u64GuestCr0 &= fZapCr0;
5744	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5745
5746	/* Commit the CR0 and related fields to the guest VMCS. */
5747	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5748	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5749	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5750	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5751	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5752	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5753	AssertRCReturn(rc, rc);
5754
5755	/* Update our caches. */
5756	pVmcsInfo->u32ProcCtls = uProcCtls;
5757	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5758
5759	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5760	}
5761	else
5762	{
5763	/*
5764	* With nested-guests, we may have extended the guest/host mask here since we
5765	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5766	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5767	* originally supplied. We must copy those bits from the nested-guest CR0 into
5768	* the nested-guest CR0 read-shadow.
5769	*/
5770	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5771	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5772	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5773	Assert(!RT_HI_U32(u64GuestCr0));
5774	Assert(u64GuestCr0 & X86_CR0_NE);
5775
5776	/*
5777	* Apply the hardware specified fixed CR0 bits and enable caching.
5778	* Note! We could be altering our VMX emulation's fixed bits. We thus
5779	* need to re-apply them while importing CR0.
5780	*/
5781	u64GuestCr0 \|= fSetCr0;
5782	u64GuestCr0 &= fZapCr0;
5783	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5784
5785	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5786	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5787	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5788	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5789	AssertRCReturn(rc, rc);
5790
5791	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5792	}
5793
5794	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5795	}
5796
5797	return VINF_SUCCESS;
5798	}
5799
5800
5801	/**
5802	* Exports the guest control registers (CR3, CR4) into the guest-state area
5803	* in the VMCS.
5804	*
5805	* @returns VBox strict status code.
5806	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5807	* without unrestricted guest access and the VMMDev is not presently
5808	* mapped (e.g. EFI32).
5809	*
5810	* @param pVCpu The cross context virtual CPU structure.
5811	* @param pVmxTransient The VMX-transient structure.
5812	*
5813	* @remarks No-long-jump zone!!!
5814	*/
5815	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5816	{
5817	int rc = VINF_SUCCESS;
5818	PVM pVM = pVCpu->CTX_SUFF(pVM);
5819
5820	/*
5821	* Guest CR2.
5822	* It's always loaded in the assembler code. Nothing to do here.
5823	*/
5824
5825	/*
5826	* Guest CR3.
5827	*/
5828	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5829	{
5830	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5831
5832	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5833	if (pVM->hm.s.fNestedPaging)
5834	{
5835	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5836	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5837
5838	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5839	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5840	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5841	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5842
5843	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5844	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5845	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5846
5847	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5848	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5849	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5850	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5851	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5852	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5853	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5854
5855	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5856	AssertRCReturn(rc, rc);
5857
5858	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5859	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5860	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5861	{
5862	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5863	if (CPUMIsGuestInPAEModeEx(pCtx))
5864	{
5865	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5866	AssertRCReturn(rc, rc);
5867	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5868	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5869	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5870	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5871	AssertRCReturn(rc, rc);
5872	}
5873
5874	/*
5875	* The guest's view of its CR3 is unblemished with nested paging when the
5876	* guest is using paging or we have unrestricted guest execution to handle
5877	* the guest when it's not using paging.
5878	*/
5879	GCPhysGuestCr3 = pCtx->cr3;
5880	}
5881	else
5882	{
5883	/*
5884	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5885	* thinks it accesses physical memory directly, we use our identity-mapped
5886	* page table to map guest-linear to guest-physical addresses. EPT takes care
5887	* of translating it to host-physical addresses.
5888	*/
5889	RTGCPHYS GCPhys;
5890	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5891
5892	/* We obtain it here every time as the guest could have relocated this PCI region. */
5893	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5894	if (RT_SUCCESS(rc))
5895	{ /* likely */ }
5896	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5897	{
5898	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5899	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5900	}
5901	else
5902	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5903
5904	GCPhysGuestCr3 = GCPhys;
5905	}
5906
5907	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5908	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5909	AssertRCReturn(rc, rc);
5910	}
5911	else
5912	{
5913	/* Non-nested paging case, just use the hypervisor's CR3. */
5914	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5915
5916	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5917	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5918	AssertRCReturn(rc, rc);
5919	}
5920
5921	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5922	}
5923
5924	/*
5925	* Guest CR4.
5926	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5927	*/
5928	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5929	{
5930	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5931	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5932
5933	/*
5934	* Figure out fixed CR4 bits in VMX operation.
5935	*/
5936	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5937	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5938	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5939
5940	/*
5941	* With nested-guests, we may have extended the guest/host mask here (since we
5942	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5943	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5944	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5945	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5946	*/
5947	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5948	uint64_t u64GuestCr4 = pCtx->cr4;
5949	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5950	? pCtx->cr4
5951	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5952	Assert(!RT_HI_U32(u64GuestCr4));
5953
5954	/*
5955	* Setup VT-x's view of the guest CR4.
5956	*
5957	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5958	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5959	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5960	*
5961	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5962	*/
5963	if (pVmcsInfo->RealMode.fRealOnV86Active)
5964	{
5965	Assert(pVM->hm.s.vmx.pRealModeTSS);
5966	Assert(PDMVmmDevHeapIsEnabled(pVM));
5967	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5968	}
5969
5970	if (pVM->hm.s.fNestedPaging)
5971	{
5972	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5973	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5974	{
5975	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5976	u64GuestCr4 \|= X86_CR4_PSE;
5977	/* Our identity mapping is a 32-bit page directory. */
5978	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5979	}
5980	/* else use guest CR4.*/
5981	}
5982	else
5983	{
5984	Assert(!pVmxTransient->fIsNestedGuest);
5985
5986	/*
5987	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5988	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5989	*/
5990	switch (pVCpu->hm.s.enmShadowMode)
5991	{
5992	case PGMMODE_REAL: /* Real-mode. */
5993	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5994	case PGMMODE_32_BIT: /* 32-bit paging. */
5995	{
5996	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5997	break;
5998	}
5999
6000	case PGMMODE_PAE: /* PAE paging. */
6001	case PGMMODE_PAE_NX: /* PAE paging with NX. */
6002	{
6003	u64GuestCr4 \|= X86_CR4_PAE;
6004	break;
6005	}
6006
6007	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
6008	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
6009	#ifdef VBOX_ENABLE_64_BITS_GUESTS
6010	break;
6011	#endif
6012	default:
6013	AssertFailed();
6014	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6015	}
6016	}
6017
6018	/*
6019	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
6020	* Note! For nested-guests, we could be altering our VMX emulation's
6021	* fixed bits. We thus need to re-apply them while importing CR4.
6022	*/
6023	u64GuestCr4 \|= fSetCr4;
6024	u64GuestCr4 &= fZapCr4;
6025
6026	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
6027	/** @todo Fix to 64-bit when we drop 32-bit. */
6028	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
6029	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
6030	AssertRCReturn(rc, rc);
6031
6032	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
6033	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6034
6035	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
6036
6037	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
6038	}
6039	return rc;
6040	}
6041
6042
6043	/**
6044	* Exports the guest debug registers into the guest-state area in the VMCS.
6045	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
6046	*
6047	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
6048	*
6049	* @returns VBox status code.
6050	* @param pVCpu The cross context virtual CPU structure.
6051	* @param pVmxTransient The VMX-transient structure.
6052	*
6053	* @remarks No-long-jump zone!!!
6054	*/
6055	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6056	{
6057	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6058
6059	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
6060	* stepping. */
6061	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6062	if (pVmxTransient->fIsNestedGuest)
6063	{
6064	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
6065	AssertRCReturn(rc, rc);
6066	return VINF_SUCCESS;
6067	}
6068
6069	#ifdef VBOX_STRICT
6070	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
6071	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
6072	{
6073	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
6074	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
6075	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
6076	}
6077	#endif
6078
6079	bool fSteppingDB = false;
6080	bool fInterceptMovDRx = false;
6081	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6082	if (pVCpu->hm.s.fSingleInstruction)
6083	{
6084	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
6085	PVM pVM = pVCpu->CTX_SUFF(pVM);
6086	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
6087	{
6088	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
6089	Assert(fSteppingDB == false);
6090	}
6091	else
6092	{
6093	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
6094	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
6095	pVCpu->hm.s.fClearTrapFlag = true;
6096	fSteppingDB = true;
6097	}
6098	}
6099
6100	uint32_t u32GuestDr7;
6101	if ( fSteppingDB
6102	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
6103	{
6104	/*
6105	* Use the combined guest and host DRx values found in the hypervisor register set
6106	* because the hypervisor debugger has breakpoints active or someone is single stepping
6107	* on the host side without a monitor trap flag.
6108	*
6109	* Note! DBGF expects a clean DR6 state before executing guest code.
6110	*/
6111	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6112	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6113	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
6114	{
6115	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6116	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
6117	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
6118	}
6119	else
6120	#endif
6121	if (!CPUMIsHyperDebugStateActive(pVCpu))
6122	{
6123	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6124	Assert(CPUMIsHyperDebugStateActive(pVCpu));
6125	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6126	}
6127
6128	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
6129	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
6130	pVCpu->hm.s.fUsingHyperDR7 = true;
6131	fInterceptMovDRx = true;
6132	}
6133	else
6134	{
6135	/*
6136	* If the guest has enabled debug registers, we need to load them prior to
6137	* executing guest code so they'll trigger at the right time.
6138	*/
6139	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6140	{
6141	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6142	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6143	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
6144	{
6145	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6146	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
6147	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
6148	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6149	}
6150	else
6151	#endif
6152	if (!CPUMIsGuestDebugStateActive(pVCpu))
6153	{
6154	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6155	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6156	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6157	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6158	}
6159	Assert(!fInterceptMovDRx);
6160	}
6161	/*
6162	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6163	* must intercept #DB in order to maintain a correct DR6 guest value, and
6164	* because we need to intercept it to prevent nested #DBs from hanging the
6165	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6166	*/
6167	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6168	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
6169	&& !CPUMIsGuestDebugStateActive(pVCpu))
6170	#else
6171	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6172	#endif
6173	{
6174	fInterceptMovDRx = true;
6175	}
6176
6177	/* Update DR7 with the actual guest value. */
6178	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6179	pVCpu->hm.s.fUsingHyperDR7 = false;
6180	}
6181
6182	if (fInterceptMovDRx)
6183	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6184	else
6185	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6186
6187	/*
6188	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6189	* monitor-trap flag and update our cache.
6190	*/
6191	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6192	{
6193	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6194	AssertRCReturn(rc2, rc2);
6195	pVmcsInfo->u32ProcCtls = uProcCtls;
6196	}
6197
6198	/*
6199	* Update guest DR7.
6200	*/
6201	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6202	AssertRCReturn(rc, rc);
6203
6204	/*
6205	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6206	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6207	*
6208	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6209	*/
6210	if (fSteppingDB)
6211	{
6212	Assert(pVCpu->hm.s.fSingleInstruction);
6213	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6214
6215	uint32_t fIntrState = 0;
6216	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6217	AssertRCReturn(rc, rc);
6218
6219	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6220	{
6221	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6222	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6223	AssertRCReturn(rc, rc);
6224	}
6225	}
6226
6227	return VINF_SUCCESS;
6228	}
6229
6230
6231	#ifdef VBOX_STRICT
6232	/**
6233	* Strict function to validate segment registers.
6234	*
6235	* @param pVCpu The cross context virtual CPU structure.
6236	* @param pVmcsInfo The VMCS info. object.
6237	*
6238	* @remarks Will import guest CR0 on strict builds during validation of
6239	* segments.
6240	*/
6241	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6242	{
6243	/*
6244	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6245	*
6246	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6247	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6248	* unusable bit and doesn't change the guest-context value.
6249	*/
6250	PVM pVM = pVCpu->CTX_SUFF(pVM);
6251	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6252	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6253	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6254	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6255	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6256	{
6257	/* Protected mode checks */
6258	/* CS */
6259	Assert(pCtx->cs.Attr.n.u1Present);
6260	Assert(!(pCtx->cs.Attr.u & 0xf00));
6261	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6262	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6263	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6264	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6265	\|\| (pCtx->cs.Attr.n.u1Granularity));
6266	/* CS cannot be loaded with NULL in protected mode. */
6267	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6268	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6269	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6270	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6271	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6272	else
6273	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6274	/* SS */
6275	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6276	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6277	if ( !(pCtx->cr0 & X86_CR0_PE)
6278	\|\| pCtx->cs.Attr.n.u4Type == 3)
6279	{
6280	Assert(!pCtx->ss.Attr.n.u2Dpl);
6281	}
6282	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6283	{
6284	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6285	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6286	Assert(pCtx->ss.Attr.n.u1Present);
6287	Assert(!(pCtx->ss.Attr.u & 0xf00));
6288	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6289	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6290	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6291	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6292	\|\| (pCtx->ss.Attr.n.u1Granularity));
6293	}
6294	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6295	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6296	{
6297	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6298	Assert(pCtx->ds.Attr.n.u1Present);
6299	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6300	Assert(!(pCtx->ds.Attr.u & 0xf00));
6301	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6302	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6303	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6304	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6305	\|\| (pCtx->ds.Attr.n.u1Granularity));
6306	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6307	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6308	}
6309	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6310	{
6311	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6312	Assert(pCtx->es.Attr.n.u1Present);
6313	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6314	Assert(!(pCtx->es.Attr.u & 0xf00));
6315	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6316	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6317	\|\| !(pCtx->es.Attr.n.u1Granularity));
6318	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6319	\|\| (pCtx->es.Attr.n.u1Granularity));
6320	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6321	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6322	}
6323	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6324	{
6325	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6326	Assert(pCtx->fs.Attr.n.u1Present);
6327	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6328	Assert(!(pCtx->fs.Attr.u & 0xf00));
6329	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6330	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6331	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6332	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6333	\|\| (pCtx->fs.Attr.n.u1Granularity));
6334	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6335	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6336	}
6337	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6338	{
6339	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6340	Assert(pCtx->gs.Attr.n.u1Present);
6341	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6342	Assert(!(pCtx->gs.Attr.u & 0xf00));
6343	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6344	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6345	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6346	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6347	\|\| (pCtx->gs.Attr.n.u1Granularity));
6348	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6349	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6350	}
6351	/* 64-bit capable CPUs. */
6352	# if HC_ARCH_BITS == 64
6353	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6354	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6355	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6356	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6357	# endif
6358	}
6359	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6360	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6361	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6362	{
6363	/* Real and v86 mode checks. */
6364	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6365	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6366	if (pVmcsInfo->RealMode.fRealOnV86Active)
6367	{
6368	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6369	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6370	}
6371	else
6372	{
6373	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6374	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6375	}
6376
6377	/* CS */
6378	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6379	Assert(pCtx->cs.u32Limit == 0xffff);
6380	Assert(u32CSAttr == 0xf3);
6381	/* SS */
6382	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6383	Assert(pCtx->ss.u32Limit == 0xffff);
6384	Assert(u32SSAttr == 0xf3);
6385	/* DS */
6386	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6387	Assert(pCtx->ds.u32Limit == 0xffff);
6388	Assert(u32DSAttr == 0xf3);
6389	/* ES */
6390	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6391	Assert(pCtx->es.u32Limit == 0xffff);
6392	Assert(u32ESAttr == 0xf3);
6393	/* FS */
6394	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6395	Assert(pCtx->fs.u32Limit == 0xffff);
6396	Assert(u32FSAttr == 0xf3);
6397	/* GS */
6398	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6399	Assert(pCtx->gs.u32Limit == 0xffff);
6400	Assert(u32GSAttr == 0xf3);
6401	/* 64-bit capable CPUs. */
6402	# if HC_ARCH_BITS == 64
6403	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6404	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6405	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6406	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6407	# endif
6408	}
6409	}
6410	#endif /* VBOX_STRICT */
6411
6412
6413	/**
6414	* Exports a guest segment register into the guest-state area in the VMCS.
6415	*
6416	* @returns VBox status code.
6417	* @param pVCpu The cross context virtual CPU structure.
6418	* @param pVmcsInfo The VMCS info. object.
6419	* @param iSegReg The segment register number (X86_SREG_XXX).
6420	* @param pSelReg Pointer to the segment selector.
6421	*
6422	* @remarks No-long-jump zone!!!
6423	*/
6424	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6425	{
6426	Assert(iSegReg < X86_SREG_COUNT);
6427	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6428	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6429	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6430	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6431
6432	uint32_t u32Access = pSelReg->Attr.u;
6433	if (pVmcsInfo->RealMode.fRealOnV86Active)
6434	{
6435	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6436	u32Access = 0xf3;
6437	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6438	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6439	RT_NOREF_PV(pVCpu);
6440	}
6441	else
6442	{
6443	/*
6444	* The way to differentiate between whether this is really a null selector or was just
6445	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6446	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6447	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6448	* NULL selectors loaded in protected-mode have their attribute as 0.
6449	*/
6450	if (!u32Access)
6451	u32Access = X86DESCATTR_UNUSABLE;
6452	}
6453
6454	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6455	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6456	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6457
6458	/*
6459	* Commit it to the VMCS.
6460	*/
6461	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6462	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6463	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6464	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6465	AssertRCReturn(rc, rc);
6466	return rc;
6467	}
6468
6469
6470	/**
6471	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6472	* area in the VMCS.
6473	*
6474	* @returns VBox status code.
6475	* @param pVCpu The cross context virtual CPU structure.
6476	* @param pVmxTransient The VMX-transient structure.
6477	*
6478	* @remarks Will import guest CR0 on strict builds during validation of
6479	* segments.
6480	* @remarks No-long-jump zone!!!
6481	*/
6482	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6483	{
6484	int rc = VERR_INTERNAL_ERROR_5;
6485	PVM pVM = pVCpu->CTX_SUFF(pVM);
6486	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6487	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6488
6489	/*
6490	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6491	*/
6492	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6493	{
6494	#ifdef VBOX_WITH_REM
6495	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6496	{
6497	Assert(!pVmxTransient->fIsNestedGuest);
6498	Assert(pVM->hm.s.vmx.pRealModeTSS);
6499	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6500	if ( pVmcsInfo->fWasInRealMode
6501	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6502	{
6503	/*
6504	* Notify the recompiler must flush its code-cache as the guest -may-
6505	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6506	*/
6507	REMFlushTBs(pVM);
6508	Log4Func(("Switch to protected mode detected!\n"));
6509	pVmcsInfo->fWasInRealMode = false;
6510	}
6511	}
6512	#endif
6513	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6514	{
6515	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6516	if (pVmcsInfo->RealMode.fRealOnV86Active)
6517	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6518	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6519	AssertRCReturn(rc, rc);
6520	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6521	}
6522
6523	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6524	{
6525	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6526	if (pVmcsInfo->RealMode.fRealOnV86Active)
6527	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6528	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6529	AssertRCReturn(rc, rc);
6530	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6531	}
6532
6533	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6534	{
6535	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6536	if (pVmcsInfo->RealMode.fRealOnV86Active)
6537	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6538	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6539	AssertRCReturn(rc, rc);
6540	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6541	}
6542
6543	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6544	{
6545	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6546	if (pVmcsInfo->RealMode.fRealOnV86Active)
6547	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6548	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6549	AssertRCReturn(rc, rc);
6550	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6551	}
6552
6553	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6554	{
6555	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6556	if (pVmcsInfo->RealMode.fRealOnV86Active)
6557	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6558	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6559	AssertRCReturn(rc, rc);
6560	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6561	}
6562
6563	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6564	{
6565	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6566	if (pVmcsInfo->RealMode.fRealOnV86Active)
6567	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6568	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6569	AssertRCReturn(rc, rc);
6570	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6571	}
6572
6573	#ifdef VBOX_STRICT
6574	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6575	#endif
6576	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6577	pCtx->cs.Attr.u));
6578	}
6579
6580	/*
6581	* Guest TR.
6582	*/
6583	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6584	{
6585	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6586
6587	/*
6588	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6589	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6590	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6591	*/
6592	uint16_t u16Sel;
6593	uint32_t u32Limit;
6594	uint64_t u64Base;
6595	uint32_t u32AccessRights;
6596	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6597	{
6598	u16Sel = pCtx->tr.Sel;
6599	u32Limit = pCtx->tr.u32Limit;
6600	u64Base = pCtx->tr.u64Base;
6601	u32AccessRights = pCtx->tr.Attr.u;
6602	}
6603	else
6604	{
6605	Assert(!pVmxTransient->fIsNestedGuest);
6606	Assert(pVM->hm.s.vmx.pRealModeTSS);
6607	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6608
6609	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6610	RTGCPHYS GCPhys;
6611	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6612	AssertRCReturn(rc, rc);
6613
6614	X86DESCATTR DescAttr;
6615	DescAttr.u = 0;
6616	DescAttr.n.u1Present = 1;
6617	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6618
6619	u16Sel = 0;
6620	u32Limit = HM_VTX_TSS_SIZE;
6621	u64Base = GCPhys;
6622	u32AccessRights = DescAttr.u;
6623	}
6624
6625	/* Validate. */
6626	Assert(!(u16Sel & RT_BIT(2)));
6627	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6628	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6629	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6630	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6631	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6632	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6633	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6634	Assert( (u32Limit & 0xfff) == 0xfff
6635	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6636	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6637	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6638
6639	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6640	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6641	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6642	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6643	AssertRCReturn(rc, rc);
6644
6645	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6646	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6647	}
6648
6649	/*
6650	* Guest GDTR.
6651	*/
6652	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6653	{
6654	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6655
6656	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6657	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6658	AssertRCReturn(rc, rc);
6659
6660	/* Validate. */
6661	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6662
6663	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6664	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6665	}
6666
6667	/*
6668	* Guest LDTR.
6669	*/
6670	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6671	{
6672	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6673
6674	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6675	uint32_t u32Access;
6676	if ( !pVmxTransient->fIsNestedGuest
6677	&& !pCtx->ldtr.Attr.u)
6678	u32Access = X86DESCATTR_UNUSABLE;
6679	else
6680	u32Access = pCtx->ldtr.Attr.u;
6681
6682	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6683	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6684	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6685	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6686	AssertRCReturn(rc, rc);
6687
6688	/* Validate. */
6689	if (!(u32Access & X86DESCATTR_UNUSABLE))
6690	{
6691	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6692	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6693	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6694	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6695	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6696	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6697	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6698	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6699	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6700	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6701	}
6702
6703	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6704	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6705	}
6706
6707	/*
6708	* Guest IDTR.
6709	*/
6710	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6711	{
6712	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6713
6714	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6715	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6716	AssertRCReturn(rc, rc);
6717
6718	/* Validate. */
6719	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6720
6721	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6722	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6723	}
6724
6725	return VINF_SUCCESS;
6726	}
6727
6728
6729	/**
6730	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6731	* areas.
6732	*
6733	* These MSRs will automatically be loaded to the host CPU on every successful
6734	* VM-entry and stored from the host CPU on every successful VM-exit.
6735	*
6736	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6737	* actual host MSR values are not- updated here for performance reasons. See
6738	* hmR0VmxExportHostMsrs().
6739	*
6740	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6741	*
6742	* @returns VBox status code.
6743	* @param pVCpu The cross context virtual CPU structure.
6744	* @param pVmxTransient The VMX-transient structure.
6745	*
6746	* @remarks No-long-jump zone!!!
6747	*/
6748	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6749	{
6750	AssertPtr(pVCpu);
6751	AssertPtr(pVmxTransient);
6752
6753	PVM pVM = pVCpu->CTX_SUFF(pVM);
6754	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6755
6756	/*
6757	* MSRs that we use the auto-load/store MSR area in the VMCS.
6758	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6759	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6760	*
6761	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6762	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6763	* emulation, nothing to do here.
6764	*/
6765	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6766	{
6767	if ( !pVmxTransient->fIsNestedGuest
6768	&& pVM->hm.s.fAllow64BitGuests)
6769	{
6770	#if HC_ARCH_BITS == 32
6771	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6772	Assert(!pVmxTransient->fIsNestedGuest);
6773
6774	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6775	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6776	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6777	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6778	AssertRCReturn(rc, rc);
6779	#endif
6780	}
6781	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6782	}
6783
6784	/*
6785	* Guest Sysenter MSRs.
6786	*/
6787	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6788	{
6789	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6790
6791	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6792	{
6793	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6794	AssertRCReturn(rc, rc);
6795	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6796	}
6797
6798	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6799	{
6800	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6801	AssertRCReturn(rc, rc);
6802	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6803	}
6804
6805	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6806	{
6807	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6808	AssertRCReturn(rc, rc);
6809	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6810	}
6811	}
6812
6813	/*
6814	* Guest/host EFER MSR.
6815	*/
6816	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6817	{
6818	/* Whether we are using the VMCS to swap the EFER MSR must have been
6819	determined earlier while exporting VM-entry/VM-exit controls. */
6820	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6821	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6822
6823	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6824	{
6825	/*
6826	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6827	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6828	*/
6829	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6830	{
6831	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6832	AssertRCReturn(rc, rc);
6833	}
6834	else
6835	{
6836	/*
6837	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6838	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6839	*/
6840	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6841	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6842	AssertRCReturn(rc, rc);
6843	}
6844	}
6845	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6846	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6847
6848	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6849	}
6850
6851	/*
6852	* Other MSRs.
6853	* Speculation Control (R/W).
6854	*/
6855	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6856	{
6857	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6858	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6859	{
6860	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6861	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6862	AssertRCReturn(rc, rc);
6863	}
6864	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6865	}
6866
6867	return VINF_SUCCESS;
6868	}
6869
6870
6871	/**
6872	* Selects up the appropriate function to run guest code.
6873	*
6874	* @returns VBox status code.
6875	* @param pVCpu The cross context virtual CPU structure.
6876	* @param pVmxTransient The VMX-transient structure.
6877	*
6878	* @remarks No-long-jump zone!!!
6879	*/
6880	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6881	{
6882	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6883	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6884
6885	if (CPUMIsGuestInLongModeEx(pCtx))
6886	{
6887	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6888	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6889	#endif
6890	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6891	#if HC_ARCH_BITS == 32
6892	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6893	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6894	{
6895	#ifdef VBOX_STRICT
6896	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6897	{
6898	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6899	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6900	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6901	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6902	("fCtxChanged=%#RX64\n", fCtxChanged));
6903	}
6904	#endif
6905	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6906
6907	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6908	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6909	pVmcsInfo->fSwitchedTo64on32 = true;
6910	Log4Func(("Selected 64-bit switcher\n"));
6911	}
6912	#else
6913	/* 64-bit host. */
6914	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6915	#endif
6916	}
6917	else
6918	{
6919	/* Guest is not in long mode, use the 32-bit handler. */
6920	#if HC_ARCH_BITS == 32
6921	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6922	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6923	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6924	{
6925	# ifdef VBOX_STRICT
6926	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6927	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6928	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6929	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6930	("fCtxChanged=%#RX64\n", fCtxChanged));
6931	# endif
6932	}
6933	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6934	/*
6935	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6936	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6937	* switcher flag now because we know the guest is in a sane state where it's safe
6938	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6939	* the much faster 32-bit switcher again.
6940	*/
6941	if (!pVmcsInfo->fSwitchedTo64on32)
6942	{
6943	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6944	Log4Func(("Selected 32-bit switcher\n"));
6945	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6946	}
6947	else
6948	{
6949	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6950	if ( pVmcsInfo->RealMode.fRealOnV86Active
6951	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6952	{
6953	pVmcsInfo->fSwitchedTo64on32 = false;
6954	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6956	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6957	\| HM_CHANGED_HOST_CONTEXT);
6958	Log4Func(("Selected 32-bit switcher (safe)\n"));
6959	}
6960	}
6961	# else
6962	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6963	# endif
6964	#else
6965	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6966	#endif
6967	}
6968	Assert(pVmcsInfo->pfnStartVM);
6969	return VINF_SUCCESS;
6970	}
6971
6972
6973	/**
6974	* Wrapper for running the guest code in VT-x.
6975	*
6976	* @returns VBox status code, no informational status codes.
6977	* @param pVCpu The cross context virtual CPU structure.
6978	* @param pVmxTransient The VMX-transient structure.
6979	*
6980	* @remarks No-long-jump zone!!!
6981	*/
6982	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6983	{
6984	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6985	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6986	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6987
6988	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6989
6990	/*
6991	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6992	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6993	* callee-saved and thus the need for this XMM wrapper.
6994	*
6995	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6996	*/
6997	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6998	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6999	PVM pVM = pVCpu->CTX_SUFF(pVM);
7000	#ifdef VBOX_WITH_KERNEL_USING_XMM
7001	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
7002	#else
7003	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
7004	#endif
7005	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
7006	return rc;
7007	}
7008
7009
7010	/**
7011	* Reports world-switch error and dumps some useful debug info.
7012	*
7013	* @param pVCpu The cross context virtual CPU structure.
7014	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
7015	* @param pVmxTransient The VMX-transient structure (only
7016	* exitReason updated).
7017	*/
7018	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
7019	{
7020	Assert(pVCpu);
7021	Assert(pVmxTransient);
7022	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7023
7024	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
7025	switch (rcVMRun)
7026	{
7027	case VERR_VMX_INVALID_VMXON_PTR:
7028	AssertFailed();
7029	break;
7030	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
7031	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
7032	{
7033	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
7034	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
7035	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
7036	AssertRC(rc);
7037
7038	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7039	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
7040	Cannot do it here as we may have been long preempted. */
7041
7042	#ifdef VBOX_STRICT
7043	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7044	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
7045	pVmxTransient->uExitReason));
7046	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
7047	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
7048	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
7049	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
7050	else
7051	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
7052	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
7053	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
7054
7055	/* VMX control bits. */
7056	uint32_t u32Val;
7057	uint64_t u64Val;
7058	RTHCUINTREG uHCReg;
7059	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
7060	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
7061	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
7062	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
7063	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
7064	{
7065	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
7066	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
7067	}
7068	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
7069	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
7070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
7071	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
7072	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
7073	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
7074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
7075	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
7076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
7077	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
7078	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
7079	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
7080	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
7081	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
7082	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
7083	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
7084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7085	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
7086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7087	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
7088	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
7089	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
7090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
7091	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
7092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
7093	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
7094	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7095	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7096	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7097	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7098	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7099	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7100	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7101	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7102	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7103	{
7104	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7105	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7106	}
7107
7108	/* Guest bits. */
7109	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
7110	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
7111	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
7112	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
7113	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
7114	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
7115	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
7116	{
7117	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
7118	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
7119	}
7120
7121	/* Host bits. */
7122	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
7123	Log4(("Host CR0 %#RHr\n", uHCReg));
7124	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
7125	Log4(("Host CR3 %#RHr\n", uHCReg));
7126	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
7127	Log4(("Host CR4 %#RHr\n", uHCReg));
7128
7129	RTGDTR HostGdtr;
7130	PCX86DESCHC pDesc;
7131	ASMGetGDTR(&HostGdtr);
7132	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
7133	Log4(("Host CS %#08x\n", u32Val));
7134	if (u32Val < HostGdtr.cbGdt)
7135	{
7136	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7137	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
7138	}
7139
7140	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
7141	Log4(("Host DS %#08x\n", u32Val));
7142	if (u32Val < HostGdtr.cbGdt)
7143	{
7144	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7145	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
7146	}
7147
7148	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
7149	Log4(("Host ES %#08x\n", u32Val));
7150	if (u32Val < HostGdtr.cbGdt)
7151	{
7152	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7153	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
7154	}
7155
7156	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
7157	Log4(("Host FS %#08x\n", u32Val));
7158	if (u32Val < HostGdtr.cbGdt)
7159	{
7160	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7161	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
7162	}
7163
7164	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
7165	Log4(("Host GS %#08x\n", u32Val));
7166	if (u32Val < HostGdtr.cbGdt)
7167	{
7168	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7169	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
7170	}
7171
7172	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
7173	Log4(("Host SS %#08x\n", u32Val));
7174	if (u32Val < HostGdtr.cbGdt)
7175	{
7176	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7177	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
7178	}
7179
7180	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
7181	Log4(("Host TR %#08x\n", u32Val));
7182	if (u32Val < HostGdtr.cbGdt)
7183	{
7184	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7185	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
7186	}
7187
7188	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
7189	Log4(("Host TR Base %#RHv\n", uHCReg));
7190	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
7191	Log4(("Host GDTR Base %#RHv\n", uHCReg));
7192	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
7193	Log4(("Host IDTR Base %#RHv\n", uHCReg));
7194	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
7195	Log4(("Host SYSENTER CS %#08x\n", u32Val));
7196	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
7197	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
7198	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
7199	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
7200	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
7201	Log4(("Host RSP %#RHv\n", uHCReg));
7202	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
7203	Log4(("Host RIP %#RHv\n", uHCReg));
7204	# if HC_ARCH_BITS == 64
7205	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
7206	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
7207	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
7208	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
7209	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
7210	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
7211	# endif
7212	#endif /* VBOX_STRICT */
7213	break;
7214	}
7215
7216	default:
7217	/* Impossible */
7218	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
7219	break;
7220	}
7221	}
7222
7223
7224	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
7225	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
7226	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
7227	# endif
7228
7229	/**
7230	* Initialize the VMCS-Read cache.
7231	*
7232	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
7233	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
7234	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
7235	* (those that have a 32-bit FULL & HIGH part).
7236	*
7237	* @param pVCpu The cross context virtual CPU structure.
7238	*/
7239	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
7240	{
7241	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
7242	do { \
7243	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
7244	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
7245	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
7246	++cReadFields; \
7247	} while (0)
7248
7249	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7250	uint32_t cReadFields = 0;
7251
7252	/*
7253	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
7254	* and serve to indicate exceptions to the rules.
7255	*/
7256
7257	/* Guest-natural selector base fields. */
7258	#if 0
7259	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
7260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
7261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
7262	#endif
7263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
7264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
7265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
7266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
7267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
7268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
7269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
7270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
7271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
7272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
7273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
7274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
7275	#if 0
7276	/* Unused natural width guest-state fields. */
7277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7279	#endif
7280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7282
7283	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7284	these 64-bit fields (using "FULL" and "HIGH" fields). */
7285	#if 0
7286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7295	#endif
7296
7297	/* Natural width guest-state fields. */
7298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7300
7301	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7302	{
7303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7304	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7305	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7306	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7307	}
7308	else
7309	{
7310	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7311	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7312	}
7313
7314	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7315	}
7316
7317
7318	/**
7319	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7320	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7321	* darwin, running 64-bit guests).
7322	*
7323	* @returns VBox status code.
7324	* @param pVCpu The cross context virtual CPU structure.
7325	* @param idxField The VMCS field encoding.
7326	* @param u64Val 16, 32 or 64-bit value.
7327	*/
7328	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7329	{
7330	AssertPtr(pVCpu);
7331	int rc;
7332	switch (idxField)
7333	{
7334	/*
7335	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7336	*/
7337	/* 64-bit Control fields. */
7338	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7339	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7340	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7341	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7342	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7343	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7344	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7345	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7346	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7347	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7348	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7349	case VMX_VMCS64_CTRL_EPTP_FULL:
7350	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7351	/* 64-bit Guest-state fields. */
7352	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7353	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7354	case VMX_VMCS64_GUEST_PAT_FULL:
7355	case VMX_VMCS64_GUEST_EFER_FULL:
7356	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7357	case VMX_VMCS64_GUEST_PDPTE0_FULL:
7358	case VMX_VMCS64_GUEST_PDPTE1_FULL:
7359	case VMX_VMCS64_GUEST_PDPTE2_FULL:
7360	case VMX_VMCS64_GUEST_PDPTE3_FULL:
7361	/* 64-bit Host-state fields. */
7362	case VMX_VMCS64_HOST_PAT_FULL:
7363	case VMX_VMCS64_HOST_EFER_FULL:
7364	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7365	{
7366	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7367	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7368	break;
7369	}
7370
7371	/*
7372	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7373	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7374	*/
7375	/* Natural-width Guest-state fields. */
7376	case VMX_VMCS_GUEST_CR3:
7377	case VMX_VMCS_GUEST_ES_BASE:
7378	case VMX_VMCS_GUEST_CS_BASE:
7379	case VMX_VMCS_GUEST_SS_BASE:
7380	case VMX_VMCS_GUEST_DS_BASE:
7381	case VMX_VMCS_GUEST_FS_BASE:
7382	case VMX_VMCS_GUEST_GS_BASE:
7383	case VMX_VMCS_GUEST_LDTR_BASE:
7384	case VMX_VMCS_GUEST_TR_BASE:
7385	case VMX_VMCS_GUEST_GDTR_BASE:
7386	case VMX_VMCS_GUEST_IDTR_BASE:
7387	case VMX_VMCS_GUEST_RSP:
7388	case VMX_VMCS_GUEST_RIP:
7389	case VMX_VMCS_GUEST_SYSENTER_ESP:
7390	case VMX_VMCS_GUEST_SYSENTER_EIP:
7391	{
7392	if (!(RT_HI_U32(u64Val)))
7393	{
7394	/* If this field is 64-bit, VT-x will zero out the top bits. */
7395	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7396	}
7397	else
7398	{
7399	/* Assert that only the 32->64 switcher case should ever come here. */
7400	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7401	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7402	}
7403	break;
7404	}
7405
7406	default:
7407	{
7408	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7409	pVCpu->hm.s.u32HMError = idxField;
7410	rc = VERR_INVALID_PARAMETER;
7411	break;
7412	}
7413	}
7414	AssertRCReturn(rc, rc);
7415	return rc;
7416	}
7417
7418
7419	/**
7420	* Queue up a VMWRITE by using the VMCS write cache.
7421	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7422	*
7423	* @param pVCpu The cross context virtual CPU structure.
7424	* @param idxField The VMCS field encoding.
7425	* @param u64Val 16, 32 or 64-bit value.
7426	*/
7427	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7428	{
7429	AssertPtr(pVCpu);
7430	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7431
7432	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7433	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7434
7435	/* Make sure there are no duplicates. */
7436	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7437	{
7438	if (pCache->Write.aField[i] == idxField)
7439	{
7440	pCache->Write.aFieldVal[i] = u64Val;
7441	return VINF_SUCCESS;
7442	}
7443	}
7444
7445	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7446	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7447	pCache->Write.cValidEntries++;
7448	return VINF_SUCCESS;
7449	}
7450	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7451
7452
7453	/**
7454	* Sets up the usage of TSC-offsetting and updates the VMCS.
7455	*
7456	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7457	* VMX-preemption timer.
7458	*
7459	* @returns VBox status code.
7460	* @param pVCpu The cross context virtual CPU structure.
7461	* @param pVmxTransient The VMX-transient structure.
7462	*
7463	* @remarks No-long-jump zone!!!
7464	*/
7465	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7466	{
7467	bool fOffsettedTsc;
7468	bool fParavirtTsc;
7469	uint64_t uTscOffset;
7470	PVM pVM = pVCpu->CTX_SUFF(pVM);
7471	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7472
7473	if (pVM->hm.s.vmx.fUsePreemptTimer)
7474	{
7475	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7476
7477	/* Make sure the returned values have sane upper and lower boundaries. */
7478	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7479	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7480	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7481	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7482
7483	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7484	* preemption timers here. We probably need to clamp the preemption timer,
7485	* after converting the timer value to the host. */
7486	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7487	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7488	AssertRC(rc);
7489	}
7490	else
7491	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7492
7493	if (fParavirtTsc)
7494	{
7495	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7496	information before every VM-entry, hence disable it for performance sake. */
7497	#if 0
7498	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7499	AssertRC(rc);
7500	#endif
7501	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7502	}
7503
7504	if ( fOffsettedTsc
7505	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7506	{
7507	if (pVmxTransient->fIsNestedGuest)
7508	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7509	hmR0VmxSetTscOffsetVmcs(pVCpu, pVmcsInfo, uTscOffset);
7510	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7511	}
7512	else
7513	{
7514	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7515	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7516	}
7517	}
7518
7519
7520	/**
7521	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7522	* VM-exit interruption info type.
7523	*
7524	* @returns The IEM exception flags.
7525	* @param uVector The event vector.
7526	* @param uVmxEventType The VMX event type.
7527	*
7528	* @remarks This function currently only constructs flags required for
7529	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7530	* and CR2 aspects of an exception are not included).
7531	*/
7532	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7533	{
7534	uint32_t fIemXcptFlags;
7535	switch (uVmxEventType)
7536	{
7537	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7538	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7539	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7540	break;
7541
7542	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7543	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7544	break;
7545
7546	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7547	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7548	break;
7549
7550	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7551	{
7552	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7553	if (uVector == X86_XCPT_BP)
7554	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7555	else if (uVector == X86_XCPT_OF)
7556	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7557	else
7558	{
7559	fIemXcptFlags = 0;
7560	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7561	}
7562	break;
7563	}
7564
7565	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7566	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7567	break;
7568
7569	default:
7570	fIemXcptFlags = 0;
7571	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7572	break;
7573	}
7574	return fIemXcptFlags;
7575	}
7576
7577
7578	/**
7579	* Sets an event as a pending event to be injected into the guest.
7580	*
7581	* @param pVCpu The cross context virtual CPU structure.
7582	* @param u32IntInfo The VM-entry interruption-information field.
7583	* @param cbInstr The VM-entry instruction length in bytes (for software
7584	* interrupts, exceptions and privileged software
7585	* exceptions).
7586	* @param u32ErrCode The VM-entry exception error code.
7587	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7588	* page-fault.
7589	*/
7590	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7591	RTGCUINTPTR GCPtrFaultAddress)
7592	{
7593	Assert(!pVCpu->hm.s.Event.fPending);
7594	pVCpu->hm.s.Event.fPending = true;
7595	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7596	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7597	pVCpu->hm.s.Event.cbInstr = cbInstr;
7598	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7599	}
7600
7601
7602	/**
7603	* Sets an external interrupt as pending-for-injection into the VM.
7604	*
7605	* @param pVCpu The cross context virtual CPU structure.
7606	* @param u8Interrupt The external interrupt vector.
7607	*/
7608	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7609	{
7610	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7611	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7612	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7613	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7614	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7615	}
7616
7617
7618	/**
7619	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7620	*
7621	* @param pVCpu The cross context virtual CPU structure.
7622	*/
7623	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7624	{
7625	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7626	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7627	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7628	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7629	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7630	}
7631
7632
7633	/**
7634	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7635	*
7636	* @param pVCpu The cross context virtual CPU structure.
7637	*/
7638	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7639	{
7640	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7641	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7642	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7643	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7644	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7645	}
7646
7647
7648	/**
7649	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7650	*
7651	* @param pVCpu The cross context virtual CPU structure.
7652	*/
7653	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7654	{
7655	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7657	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7658	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7659	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7660	}
7661
7662
7663	/**
7664	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7665	*
7666	* @param pVCpu The cross context virtual CPU structure.
7667	*/
7668	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7669	{
7670	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7672	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7673	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7674	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7675	}
7676
7677
7678	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7679	/**
7680	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7681	*
7682	* @param pVCpu The cross context virtual CPU structure.
7683	* @param u32ErrCode The error code for the general-protection exception.
7684	*/
7685	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7686	{
7687	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7689	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7690	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7691	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7692	}
7693
7694
7695	/**
7696	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7697	*
7698	* @param pVCpu The cross context virtual CPU structure.
7699	* @param u32ErrCode The error code for the stack exception.
7700	*/
7701	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7702	{
7703	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7704	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7706	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7707	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7708	}
7709	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7710
7711
7712	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7713	{
7714	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7715
7716	/*
7717	* If VT-x marks the segment as unusable, most other bits remain undefined:
7718	* - For CS the L, D and G bits have meaning.
7719	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7720	* - For the remaining data segments no bits are defined.
7721	*
7722	* The present bit and the unusable bit has been observed to be set at the
7723	* same time (the selector was supposed to be invalid as we started executing
7724	* a V8086 interrupt in ring-0).
7725	*
7726	* What should be important for the rest of the VBox code, is that the P bit is
7727	* cleared. Some of the other VBox code recognizes the unusable bit, but
7728	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7729	* safe side here, we'll strip off P and other bits we don't care about. If
7730	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7731	*
7732	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7733	*/
7734	#ifdef VBOX_STRICT
7735	uint32_t const uAttr = pSelReg->Attr.u;
7736	#endif
7737
7738	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7739	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7740	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7741
7742	#ifdef VBOX_STRICT
7743	VMMRZCallRing3Disable(pVCpu);
7744	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7745	# ifdef DEBUG_bird
7746	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7747	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7748	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7749	# endif
7750	VMMRZCallRing3Enable(pVCpu);
7751	NOREF(uAttr);
7752	#endif
7753	RT_NOREF2(pVCpu, idxSel);
7754	}
7755
7756
7757	/**
7758	* Imports a guest segment register from the current VMCS into the guest-CPU
7759	* context.
7760	*
7761	* @returns VBox status code.
7762	* @param pVCpu The cross context virtual CPU structure.
7763	* @param iSegReg The segment register number (X86_SREG_XXX).
7764	*
7765	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7766	* do not log!
7767	*/
7768	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7769	{
7770	Assert(iSegReg < X86_SREG_COUNT);
7771
7772	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7773	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7774	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7775	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7776	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7777	#else
7778	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7779	#endif
7780	uint64_t u64Base;
7781	uint32_t u32Sel, u32Limit, u32Attr;
7782	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7783	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7784	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7785	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7786	if (RT_SUCCESS(rc))
7787	{
7788	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7789	pSelReg->Sel = u32Sel;
7790	pSelReg->ValidSel = u32Sel;
7791	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7792	pSelReg->u32Limit = u32Limit;
7793	pSelReg->u64Base = u64Base;
7794	pSelReg->Attr.u = u32Attr;
7795	if (u32Attr & X86DESCATTR_UNUSABLE)
7796	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7797	}
7798	return rc;
7799	}
7800
7801
7802	/**
7803	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7804	*
7805	* @returns VBox status code.
7806	* @param pVCpu The cross context virtual CPU structure.
7807	*
7808	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7809	* do not log!
7810	*/
7811	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7812	{
7813	uint64_t u64Base;
7814	uint32_t u32Sel, u32Limit, u32Attr;
7815	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7816	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7817	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7818	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7819	if (RT_SUCCESS(rc))
7820	{
7821	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7822	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7823	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7824	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7825	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7826	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7827	if (u32Attr & X86DESCATTR_UNUSABLE)
7828	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7829	}
7830	return rc;
7831	}
7832
7833
7834	/**
7835	* Imports the guest TR from the current VMCS into the guest-CPU context.
7836	*
7837	* @returns VBox status code.
7838	* @param pVCpu The cross context virtual CPU structure.
7839	*
7840	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7841	* do not log!
7842	*/
7843	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7844	{
7845	uint32_t u32Sel, u32Limit, u32Attr;
7846	uint64_t u64Base;
7847	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7848	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7849	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7850	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7851	AssertRCReturn(rc, rc);
7852
7853	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7854	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7855	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7856	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7857	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7858	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7859	/* TR is the only selector that can never be unusable. */
7860	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7861	return VINF_SUCCESS;
7862	}
7863
7864
7865	/**
7866	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7867	*
7868	* @returns VBox status code.
7869	* @param pVCpu The cross context virtual CPU structure.
7870	*
7871	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7872	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7873	* instead!!!
7874	*/
7875	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7876	{
7877	uint64_t u64Val;
7878	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7879	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7880	{
7881	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7882	if (RT_SUCCESS(rc))
7883	{
7884	pCtx->rip = u64Val;
7885	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7886	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7887	}
7888	return rc;
7889	}
7890	return VINF_SUCCESS;
7891	}
7892
7893
7894	/**
7895	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7896	*
7897	* @returns VBox status code.
7898	* @param pVCpu The cross context virtual CPU structure.
7899	* @param pVmcsInfo The VMCS info. object.
7900	*
7901	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7902	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7903	* instead!!!
7904	*/
7905	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7906	{
7907	uint32_t u32Val;
7908	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7909	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7910	{
7911	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7912	if (RT_SUCCESS(rc))
7913	{
7914	pCtx->eflags.u32 = u32Val;
7915
7916	/* Restore eflags for real-on-v86-mode hack. */
7917	if (pVmcsInfo->RealMode.fRealOnV86Active)
7918	{
7919	pCtx->eflags.Bits.u1VM = 0;
7920	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7921	}
7922	}
7923	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7924	return rc;
7925	}
7926	return VINF_SUCCESS;
7927	}
7928
7929
7930	/**
7931	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7932	* context.
7933	*
7934	* @returns VBox status code.
7935	* @param pVCpu The cross context virtual CPU structure.
7936	* @param pVmcsInfo The VMCS info. object.
7937	*
7938	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7939	* do not log!
7940	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7941	* instead!!!
7942	*/
7943	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7944	{
7945	uint32_t u32Val;
7946	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7947	if (RT_SUCCESS(rc))
7948	{
7949	if (!u32Val)
7950	{
7951	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7952	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7953
7954	CPUMSetGuestNmiBlocking(pVCpu, false);
7955	}
7956	else
7957	{
7958	/*
7959	* We must import RIP here to set our EM interrupt-inhibited state.
7960	* We also import RFLAGS as our code that evaluates pending interrupts
7961	* before VM-entry requires it.
7962	*/
7963	rc = hmR0VmxImportGuestRip(pVCpu);
7964	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7965	if (RT_SUCCESS(rc))
7966	{
7967	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7968	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7969	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7970	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7971
7972	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7973	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7974	}
7975	}
7976	}
7977	return rc;
7978	}
7979
7980
7981	/**
7982	* Worker for VMXR0ImportStateOnDemand.
7983	*
7984	* @returns VBox status code.
7985	* @param pVCpu The cross context virtual CPU structure.
7986	* @param pVmcsInfo The VMCS info. object.
7987	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7988	*/
7989	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7990	{
7991	#define VMXLOCAL_BREAK_RC(a_rc) \
7992	if (RT_SUCCESS(a_rc)) \
7993	{ } \
7994	else \
7995	break
7996
7997	int rc = VINF_SUCCESS;
7998	PVM pVM = pVCpu->CTX_SUFF(pVM);
7999	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8000	uint64_t u64Val;
8001	uint32_t u32Val;
8002
8003	/*
8004	* Note! This is hack to workaround a mysterious BSOD observed with release builds
8005	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8006	* neither are other host platforms.
8007	*
8008	* Committing this temporarily as it prevents BSOD.
8009	*
8010	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8011	*/
8012	#ifdef RT_OS_WINDOWS
8013	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
8014	return VERR_HM_IPE_1;
8015	#endif
8016
8017	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8018
8019	/*
8020	* We disable interrupts to make the updating of the state and in particular
8021	* the fExtrn modification atomic wrt to preemption hooks.
8022	*/
8023	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8024
8025	fWhat &= pCtx->fExtrn;
8026	if (fWhat)
8027	{
8028	do
8029	{
8030	if (fWhat & CPUMCTX_EXTRN_RIP)
8031	{
8032	rc = hmR0VmxImportGuestRip(pVCpu);
8033	VMXLOCAL_BREAK_RC(rc);
8034	}
8035
8036	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8037	{
8038	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8039	VMXLOCAL_BREAK_RC(rc);
8040	}
8041
8042	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8043	{
8044	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8045	VMXLOCAL_BREAK_RC(rc);
8046	}
8047
8048	if (fWhat & CPUMCTX_EXTRN_RSP)
8049	{
8050	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8051	VMXLOCAL_BREAK_RC(rc);
8052	pCtx->rsp = u64Val;
8053	}
8054
8055	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8056	{
8057	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8058	if (fWhat & CPUMCTX_EXTRN_CS)
8059	{
8060	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8061	rc \|= hmR0VmxImportGuestRip(pVCpu);
8062	if (fRealOnV86Active)
8063	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8064	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8065	}
8066	if (fWhat & CPUMCTX_EXTRN_SS)
8067	{
8068	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8069	if (fRealOnV86Active)
8070	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8071	}
8072	if (fWhat & CPUMCTX_EXTRN_DS)
8073	{
8074	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8075	if (fRealOnV86Active)
8076	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8077	}
8078	if (fWhat & CPUMCTX_EXTRN_ES)
8079	{
8080	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8081	if (fRealOnV86Active)
8082	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8083	}
8084	if (fWhat & CPUMCTX_EXTRN_FS)
8085	{
8086	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8087	if (fRealOnV86Active)
8088	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8089	}
8090	if (fWhat & CPUMCTX_EXTRN_GS)
8091	{
8092	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8093	if (fRealOnV86Active)
8094	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8095	}
8096	VMXLOCAL_BREAK_RC(rc);
8097	}
8098
8099	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8100	{
8101	if (fWhat & CPUMCTX_EXTRN_LDTR)
8102	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
8103
8104	if (fWhat & CPUMCTX_EXTRN_GDTR)
8105	{
8106	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8107	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8108	pCtx->gdtr.pGdt = u64Val;
8109	pCtx->gdtr.cbGdt = u32Val;
8110	}
8111
8112	/* Guest IDTR. */
8113	if (fWhat & CPUMCTX_EXTRN_IDTR)
8114	{
8115	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8116	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8117	pCtx->idtr.pIdt = u64Val;
8118	pCtx->idtr.cbIdt = u32Val;
8119	}
8120
8121	/* Guest TR. */
8122	if (fWhat & CPUMCTX_EXTRN_TR)
8123	{
8124	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8125	don't need to import that one. */
8126	if (!pVmcsInfo->RealMode.fRealOnV86Active)
8127	rc \|= hmR0VmxImportGuestTr(pVCpu);
8128	}
8129	VMXLOCAL_BREAK_RC(rc);
8130	}
8131
8132	if (fWhat & CPUMCTX_EXTRN_DR7)
8133	{
8134	if (!pVCpu->hm.s.fUsingHyperDR7)
8135	{
8136	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8137	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8138	VMXLOCAL_BREAK_RC(rc);
8139	pCtx->dr[7] = u32Val;
8140	}
8141	}
8142
8143	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8144	{
8145	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8146	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8147	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8148	pCtx->SysEnter.cs = u32Val;
8149	VMXLOCAL_BREAK_RC(rc);
8150	}
8151
8152	#if HC_ARCH_BITS == 64
8153	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8154	{
8155	if ( pVM->hm.s.fAllow64BitGuests
8156	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8157	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8158	}
8159
8160	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8161	{
8162	if ( pVM->hm.s.fAllow64BitGuests
8163	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8164	{
8165	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8166	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8167	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8168	}
8169	}
8170	#endif
8171
8172	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
8173	#if HC_ARCH_BITS == 32
8174	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
8175	#endif
8176	)
8177	{
8178	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8179	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8180	Assert(pMsrs);
8181	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8182	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
8183	for (uint32_t i = 0; i < cMsrs; i++)
8184	{
8185	uint32_t const idMsr = pMsrs[i].u32Msr;
8186	switch (idMsr)
8187	{
8188	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8189	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8190	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8191	#if HC_ARCH_BITS == 32
8192	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8193	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8194	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8195	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8196	#endif
8197	default:
8198	{
8199	pCtx->fExtrn = 0;
8200	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8201	ASMSetFlags(fEFlags);
8202	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8203	return VERR_HM_UNEXPECTED_LD_ST_MSR;
8204	}
8205	}
8206	}
8207	}
8208
8209	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8210	{
8211	uint64_t u64Shadow;
8212	if (fWhat & CPUMCTX_EXTRN_CR0)
8213	{
8214	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8215	* remove when we drop 32-bit host w/ 64-bit host support, see
8216	* @bugref{9180#c39}. */
8217	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8218	#if HC_ARCH_BITS == 32
8219	uint32_t u32Shadow;
8220	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8221	u64Shadow = u32Shadow;
8222	#else
8223	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8224	#endif
8225	VMXLOCAL_BREAK_RC(rc);
8226	u64Val = u32Val;
8227	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8228	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
8229	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8230	/*
8231	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
8232	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8233	*/
8234	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8235	{
8236	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8237	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8238	}
8239	#endif
8240	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8241	CPUMSetGuestCR0(pVCpu, u64Val);
8242	VMMRZCallRing3Enable(pVCpu);
8243	}
8244
8245	if (fWhat & CPUMCTX_EXTRN_CR4)
8246	{
8247	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8248	* remove when we drop 32-bit host w/ 64-bit host support, see
8249	* @bugref{9180#c39}. */
8250	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8251	#if HC_ARCH_BITS == 32
8252	uint32_t u32Shadow;
8253	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8254	u64Shadow = u32Shadow;
8255	#else
8256	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8257	#endif
8258	VMXLOCAL_BREAK_RC(rc);
8259	u64Val = u32Val;
8260	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8261	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
8262	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8263	/*
8264	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
8265	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8266	*/
8267	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8268	{
8269	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8270	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8271	}
8272	#endif
8273	pCtx->cr4 = u64Val;
8274	}
8275
8276	if (fWhat & CPUMCTX_EXTRN_CR3)
8277	{
8278	/* CR0.PG bit changes are always intercepted, so it's up to date. */
8279	if ( pVM->hm.s.vmx.fUnrestrictedGuest
8280	\|\| ( pVM->hm.s.fNestedPaging
8281	&& CPUMIsGuestPagingEnabledEx(pCtx)))
8282	{
8283	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8284	VMXLOCAL_BREAK_RC(rc);
8285	if (pCtx->cr3 != u64Val)
8286	{
8287	pCtx->cr3 = u64Val;
8288	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8289	}
8290
8291	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8292	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8293	if (CPUMIsGuestInPAEModeEx(pCtx))
8294	{
8295	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8296	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8297	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8298	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8299	VMXLOCAL_BREAK_RC(rc);
8300	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8301	}
8302	}
8303	}
8304	}
8305
8306	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8307	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
8308	{
8309	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
8310	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
8311	{
8312	Assert(CPUMIsGuestInVmxRootMode(pCtx));
8313	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
8314	VMXLOCAL_BREAK_RC(rc);
8315	}
8316
8317	# if 0
8318	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8319	* VM-exit handlers as parameters. We would handle it differently when using
8320	* the fast path. */
8321	/*
8322	* The hardware virtualization state currently consists of VMCS fields that may be
8323	* modified by execution of the nested-guest (that are not part of the general
8324	* guest state) and is visible to guest software. Hence, it is technically part of
8325	* the guest-CPU state when executing a nested-guest.
8326	*/
8327	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8328	{
8329	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8330	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8331	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8332	VMXLOCAL_BREAK_RC(rc);
8333
8334	/*
8335	* VM-entry can fail due to invalid-guest state, machine-check events and
8336	* MSR loading failures. Other than VM-exit reason and Exit qualification
8337	* all other VMCS fields are left unmodified on VM-entry failure.
8338	*
8339	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8340	*/
8341	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8342	if (!fEntryFailed)
8343	{
8344	/*
8345	* Some notes on VMCS fields that may need importing when the fast path
8346	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8347	*
8348	* Requires fixing up when using hardware-assisted VMX:
8349	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8350	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
8351	* - IDT-vectoring info: Think about this.
8352	* - IDT-vectoring error code: Think about this.
8353	*
8354	* Emulated:
8355	* - Guest-interruptiblity state: Derived from FFs and RIP.
8356	* - Guest pending debug exceptions: Derived from DR6.
8357	* - Guest activity state: Emulated from EM state.
8358	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8359	* - Entry-interrupt info: Emulated, cleared to 0.
8360	*/
8361	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8362	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8363	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8364	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8365	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8366	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8367	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8368	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8369	/** @todo NSTVMX: Save and adjust preemption timer value. */
8370	}
8371
8372	VMXLOCAL_BREAK_RC(rc);
8373	}
8374	# endif
8375	}
8376	#endif
8377	} while (0);
8378
8379	if (RT_SUCCESS(rc))
8380	{
8381	/* Update fExtrn. */
8382	pCtx->fExtrn &= ~fWhat;
8383
8384	/* If everything has been imported, clear the HM keeper bit. */
8385	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8386	{
8387	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8388	Assert(!pCtx->fExtrn);
8389	}
8390	}
8391	}
8392	else
8393	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8394
8395	ASMSetFlags(fEFlags);
8396
8397	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8398
8399	if (RT_SUCCESS(rc))
8400	{ /* likely */ }
8401	else
8402	return rc;
8403
8404	/*
8405	* Honor any pending CR3 updates.
8406	*
8407	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8408	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8409	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8410	*
8411	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8412	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8413	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8414	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8415	*
8416	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8417	*/
8418	if (VMMRZCallRing3IsEnabled(pVCpu))
8419	{
8420	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8421	{
8422	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8423	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8424	}
8425
8426	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8427	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8428
8429	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8430	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8431	}
8432
8433	return VINF_SUCCESS;
8434	#undef VMXLOCAL_BREAK_RC
8435	}
8436
8437
8438	/**
8439	* Saves the guest state from the VMCS into the guest-CPU context.
8440	*
8441	* @returns VBox status code.
8442	* @param pVCpu The cross context virtual CPU structure.
8443	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8444	*/
8445	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8446	{
8447	AssertPtr(pVCpu);
8448	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8449	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8450	}
8451
8452
8453	/**
8454	* Check per-VM and per-VCPU force flag actions that require us to go back to
8455	* ring-3 for one reason or another.
8456	*
8457	* @returns Strict VBox status code (i.e. informational status codes too)
8458	* @retval VINF_SUCCESS if we don't have any actions that require going back to
8459	* ring-3.
8460	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8461	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8462	* interrupts)
8463	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8464	* all EMTs to be in ring-3.
8465	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8466	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8467	* to the EM loop.
8468	*
8469	* @param pVCpu The cross context virtual CPU structure.
8470	* @param fStepping Whether we are single-stepping the guest using the
8471	* hypervisor debugger.
8472	*/
8473	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8474	{
8475	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8476
8477	/*
8478	* Update pending interrupts into the APIC's IRR.
8479	*/
8480	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8481	APICUpdatePendingInterrupts(pVCpu);
8482
8483	/*
8484	* Anything pending? Should be more likely than not if we're doing a good job.
8485	*/
8486	PVM pVM = pVCpu->CTX_SUFF(pVM);
8487	if ( !fStepping
8488	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8489	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8490	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8491	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8492	return VINF_SUCCESS;
8493
8494	/* Pending PGM C3 sync. */
8495	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8496	{
8497	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8498	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8499	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8500	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8501	if (rcStrict2 != VINF_SUCCESS)
8502	{
8503	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8504	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8505	return rcStrict2;
8506	}
8507	}
8508
8509	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8510	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8511	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8512	{
8513	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8514	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8515	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8516	return rc2;
8517	}
8518
8519	/* Pending VM request packets, such as hardware interrupts. */
8520	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8521	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8522	{
8523	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
8524	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8525	return VINF_EM_PENDING_REQUEST;
8526	}
8527
8528	/* Pending PGM pool flushes. */
8529	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8530	{
8531	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
8532	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8533	return VINF_PGM_POOL_FLUSH_PENDING;
8534	}
8535
8536	/* Pending DMA requests. */
8537	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8538	{
8539	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
8540	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8541	return VINF_EM_RAW_TO_R3;
8542	}
8543
8544	return VINF_SUCCESS;
8545	}
8546
8547
8548	/**
8549	* Converts any TRPM trap into a pending HM event. This is typically used when
8550	* entering from ring-3 (not longjmp returns).
8551	*
8552	* @param pVCpu The cross context virtual CPU structure.
8553	*/
8554	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8555	{
8556	Assert(TRPMHasTrap(pVCpu));
8557	Assert(!pVCpu->hm.s.Event.fPending);
8558
8559	uint8_t uVector;
8560	TRPMEVENT enmTrpmEvent;
8561	RTGCUINT uErrCode;
8562	RTGCUINTPTR GCPtrFaultAddress;
8563	uint8_t cbInstr;
8564
8565	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8566	AssertRC(rc);
8567
8568	uint32_t u32IntInfo;
8569	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8570	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
8571
8572	rc = TRPMResetTrap(pVCpu);
8573	AssertRC(rc);
8574	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8575	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8576
8577	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8578	}
8579
8580
8581	/**
8582	* Converts the pending HM event into a TRPM trap.
8583	*
8584	* @param pVCpu The cross context virtual CPU structure.
8585	*/
8586	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8587	{
8588	Assert(pVCpu->hm.s.Event.fPending);
8589
8590	/* If a trap was already pending, we did something wrong! */
8591	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8592
8593	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8594	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8595	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8596
8597	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8598
8599	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8600	AssertRC(rc);
8601
8602	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8603	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8604
8605	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8606	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8607	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
8608	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8609
8610	/* We're now done converting the pending event. */
8611	pVCpu->hm.s.Event.fPending = false;
8612	}
8613
8614
8615	/**
8616	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8617	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8618	*
8619	* @param pVCpu The cross context virtual CPU structure.
8620	* @param pVmcsInfo The VMCS info. object.
8621	*/
8622	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8623	{
8624	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8625	{
8626	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8627	{
8628	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8629	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8630	AssertRC(rc);
8631	}
8632	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8633	}
8634
8635
8636	/**
8637	* Clears the interrupt-window exiting control in the VMCS.
8638	*
8639	* @param pVmcsInfo The VMCS info. object.
8640	*/
8641	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8642	{
8643	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8644	{
8645	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8646	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8647	}
8648	return VINF_SUCCESS;
8649	}
8650
8651
8652	/**
8653	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8654	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8655	*
8656	* @param pVCpu The cross context virtual CPU structure.
8657	* @param pVmcsInfo The VMCS info. object.
8658	*/
8659	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8660	{
8661	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8662	{
8663	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8664	{
8665	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8666	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8667	AssertRC(rc);
8668	Log4Func(("Setup NMI-window exiting\n"));
8669	}
8670	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8671	}
8672
8673
8674	/**
8675	* Clears the NMI-window exiting control in the VMCS.
8676	*
8677	* @param pVmcsInfo The VMCS info. object.
8678	*/
8679	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8680	{
8681	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8682	{
8683	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8684	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8685	}
8686	return VINF_SUCCESS;
8687	}
8688
8689
8690	/**
8691	* Does the necessary state syncing before returning to ring-3 for any reason
8692	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8693	*
8694	* @returns VBox status code.
8695	* @param pVCpu The cross context virtual CPU structure.
8696	* @param fImportState Whether to import the guest state from the VMCS back
8697	* to the guest-CPU context.
8698	*
8699	* @remarks No-long-jmp zone!!!
8700	*/
8701	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8702	{
8703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8704	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8705
8706	RTCPUID const idCpu = RTMpCpuId();
8707	Log4Func(("HostCpuId=%u\n", idCpu));
8708
8709	/*
8710	* !!! IMPORTANT !!!
8711	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8712	*/
8713
8714	/* Save the guest state if necessary. */
8715	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8716	if (fImportState)
8717	{
8718	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8719	AssertRCReturn(rc, rc);
8720	}
8721
8722	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8723	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8724	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8725
8726	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8727	#ifdef VBOX_STRICT
8728	if (CPUMIsHyperDebugStateActive(pVCpu))
8729	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8730	#endif
8731	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8732	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8733	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8734
8735	#if HC_ARCH_BITS == 64
8736	/* Restore host-state bits that VT-x only restores partially. */
8737	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8738	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8739	{
8740	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8741	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8742	}
8743	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8744	#endif
8745
8746	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8747	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8748	{
8749	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8750	if (!fImportState)
8751	{
8752	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8753	AssertRCReturn(rc, rc);
8754	}
8755	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8756	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8757	}
8758	else
8759	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8760
8761	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8762	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8763
8764	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8765	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8766	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8767	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8768	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8769	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8770	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8771	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8772	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8773	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8774
8775	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8776
8777	/** @todo This partially defeats the purpose of having preemption hooks.
8778	* The problem is, deregistering the hooks should be moved to a place that
8779	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8780	* context.
8781	*/
8782	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8783	AssertRCReturn(rc, rc);
8784
8785	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8786	/*
8787	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8788	* clear a shadow VMCS before allowing that VMCS to become active on another
8789	* logical processor. We may or may not be importing guest state which clears
8790	* it, so cover for it here.
8791	*
8792	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8793	*/
8794	if ( pVmcsInfo->pvShadowVmcs
8795	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8796	{
8797	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8798	AssertRCReturn(rc, rc);
8799	}
8800
8801	/*
8802	* Flag that we need to re-import the host state if we switch to this VMCS before
8803	* executing guest or nested-guest code.
8804	*/
8805	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8806	#endif
8807
8808	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8809	NOREF(idCpu);
8810	return VINF_SUCCESS;
8811	}
8812
8813
8814	/**
8815	* Leaves the VT-x session.
8816	*
8817	* @returns VBox status code.
8818	* @param pVCpu The cross context virtual CPU structure.
8819	*
8820	* @remarks No-long-jmp zone!!!
8821	*/
8822	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8823	{
8824	HM_DISABLE_PREEMPT(pVCpu);
8825	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8826	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8827	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8828
8829	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8830	and done this from the VMXR0ThreadCtxCallback(). */
8831	if (!pVCpu->hm.s.fLeaveDone)
8832	{
8833	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8834	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8835	pVCpu->hm.s.fLeaveDone = true;
8836	}
8837	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8838
8839	/*
8840	* !!! IMPORTANT !!!
8841	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8842	*/
8843
8844	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8845	/** @todo Deregistering here means we need to VMCLEAR always
8846	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8847	* for calling VMMR0ThreadCtxHookDisable here! */
8848	VMMR0ThreadCtxHookDisable(pVCpu);
8849
8850	/* Leave HM context. This takes care of local init (term). */
8851	int rc = HMR0LeaveCpu(pVCpu);
8852
8853	HM_RESTORE_PREEMPT();
8854	return rc;
8855	}
8856
8857
8858	/**
8859	* Does the necessary state syncing before doing a longjmp to ring-3.
8860	*
8861	* @returns VBox status code.
8862	* @param pVCpu The cross context virtual CPU structure.
8863	*
8864	* @remarks No-long-jmp zone!!!
8865	*/
8866	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8867	{
8868	return hmR0VmxLeaveSession(pVCpu);
8869	}
8870
8871
8872	/**
8873	* Take necessary actions before going back to ring-3.
8874	*
8875	* An action requires us to go back to ring-3. This function does the necessary
8876	* steps before we can safely return to ring-3. This is not the same as longjmps
8877	* to ring-3, this is voluntary and prepares the guest so it may continue
8878	* executing outside HM (recompiler/IEM).
8879	*
8880	* @returns VBox status code.
8881	* @param pVCpu The cross context virtual CPU structure.
8882	* @param rcExit The reason for exiting to ring-3. Can be
8883	* VINF_VMM_UNKNOWN_RING3_CALL.
8884	*/
8885	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8886	{
8887	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8888
8889	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8890	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8891	{
8892	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8893	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8894	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8895	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8896	}
8897
8898	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8899	VMMRZCallRing3Disable(pVCpu);
8900	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8901
8902	/*
8903	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8904	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8905	*
8906	* This is because execution may continue from ring-3 and we would need to inject
8907	* the event from there (hence place it back in TRPM).
8908	*/
8909	if (pVCpu->hm.s.Event.fPending)
8910	{
8911	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8912	Assert(!pVCpu->hm.s.Event.fPending);
8913
8914	/* Clear the events from the VMCS. */
8915	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8916	AssertRCReturn(rc, rc);
8917	}
8918	#ifdef VBOX_STRICT
8919	else
8920	{
8921	/*
8922	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8923	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8924	* occasionally, see @bugref{9180#c42}.
8925	*
8926	* However, if the VM-entry failed, any VM entry-interruption info. field would
8927	* be left unmodified as the event would not have been injected to the guest. In
8928	* such cases, don't assert, we're not going to continue guest execution anyway.
8929	*/
8930	uint32_t uExitReason;
8931	uint32_t uEntryIntInfo;
8932	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8933	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8934	AssertRC(rc);
8935	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8936	}
8937	#endif
8938
8939	/*
8940	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8941	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8942	* (e.g. TPR below threshold).
8943	*/
8944	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8945	{
8946	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8947	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8948	AssertRCReturn(rc, rc);
8949	}
8950
8951	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8952	and if we're injecting an event we should have a TRPM trap pending. */
8953	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8954	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8955	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8956	#endif
8957
8958	/* Save guest state and restore host state bits. */
8959	int rc = hmR0VmxLeaveSession(pVCpu);
8960	AssertRCReturn(rc, rc);
8961	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8962
8963	/* Thread-context hooks are unregistered at this point!!! */
8964
8965	/* Sync recompiler state. */
8966	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8967	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8968	\| CPUM_CHANGED_LDTR
8969	\| CPUM_CHANGED_GDTR
8970	\| CPUM_CHANGED_IDTR
8971	\| CPUM_CHANGED_TR
8972	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8973	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8974	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8975	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8976
8977	Assert(!pVCpu->hm.s.fClearTrapFlag);
8978
8979	/* Update the exit-to-ring 3 reason. */
8980	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8981
8982	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8983	if ( rcExit != VINF_EM_RAW_INTERRUPT
8984	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8985	{
8986	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8987	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8988	}
8989
8990	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8991
8992	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8993	VMMRZCallRing3RemoveNotification(pVCpu);
8994	VMMRZCallRing3Enable(pVCpu);
8995
8996	return rc;
8997	}
8998
8999
9000	/**
9001	* VMMRZCallRing3() callback wrapper which saves the guest state before we
9002	* longjump to ring-3 and possibly get preempted.
9003	*
9004	* @returns VBox status code.
9005	* @param pVCpu The cross context virtual CPU structure.
9006	* @param enmOperation The operation causing the ring-3 longjump.
9007	* @param pvUser User argument, currently unused, NULL.
9008	*/
9009	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9010	{
9011	RT_NOREF(pvUser);
9012	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9013	{
9014	/*
9015	* !!! IMPORTANT !!!
9016	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9017	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9018	*/
9019	VMMRZCallRing3RemoveNotification(pVCpu);
9020	VMMRZCallRing3Disable(pVCpu);
9021	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9022	RTThreadPreemptDisable(&PreemptState);
9023
9024	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9025	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9026	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9027	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9028
9029	#if HC_ARCH_BITS == 64
9030	/* Restore host-state bits that VT-x only restores partially. */
9031	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9032	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9033	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9034	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9035	#endif
9036
9037	/* Restore the lazy host MSRs as we're leaving VT-x context. */
9038	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9039	hmR0VmxLazyRestoreHostMsrs(pVCpu);
9040
9041	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9042	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9043	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9044
9045	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
9046	cleared as part of importing the guest state above. */
9047	hmR0VmxClearVmcs(pVmcsInfo);
9048
9049	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9050	VMMR0ThreadCtxHookDisable(pVCpu);
9051	HMR0LeaveCpu(pVCpu);
9052	RTThreadPreemptRestore(&PreemptState);
9053	return VINF_SUCCESS;
9054	}
9055
9056	Assert(pVCpu);
9057	Assert(pvUser);
9058	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9059	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9060
9061	VMMRZCallRing3Disable(pVCpu);
9062	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9063
9064	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9065
9066	int rc = hmR0VmxLongJmpToRing3(pVCpu);
9067	AssertRCReturn(rc, rc);
9068
9069	VMMRZCallRing3Enable(pVCpu);
9070	return VINF_SUCCESS;
9071	}
9072
9073
9074	/**
9075	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9076	* stack.
9077	*
9078	* @returns Strict VBox status code (i.e. informational status codes too).
9079	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9080	* @param pVCpu The cross context virtual CPU structure.
9081	* @param uValue The value to push to the guest stack.
9082	*/
9083	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9084	{
9085	/*
9086	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9087	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9088	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9089	*/
9090	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9091	if (pCtx->sp == 1)
9092	return VINF_EM_RESET;
9093	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9094	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9095	AssertRC(rc);
9096	return rc;
9097	}
9098
9099
9100	/**
9101	* Injects an event into the guest upon VM-entry by updating the relevant fields
9102	* in the VM-entry area in the VMCS.
9103	*
9104	* @returns Strict VBox status code (i.e. informational status codes too).
9105	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9106	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9107	*
9108	* @param pVCpu The cross context virtual CPU structure.
9109	* @param pVmxTransient The VMX-transient structure.
9110	* @param pEvent The event being injected.
9111	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
9112	* will be updated if necessary. This cannot not be NULL.
9113	* @param fStepping Whether we're single-stepping guest execution and should
9114	* return VINF_EM_DBG_STEPPED if the event is injected
9115	* directly (registers modified by us, not by hardware on
9116	* VM-entry).
9117	*/
9118	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9119	uint32_t *pfIntrState)
9120	{
9121	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9122	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9123	Assert(pfIntrState);
9124
9125	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9126	uint32_t u32IntInfo = pEvent->u64IntInfo;
9127	uint32_t const u32ErrCode = pEvent->u32ErrCode;
9128	uint32_t const cbInstr = pEvent->cbInstr;
9129	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9130	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9131	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9132
9133	#ifdef VBOX_STRICT
9134	/*
9135	* Validate the error-code-valid bit for hardware exceptions.
9136	* No error codes for exceptions in real-mode.
9137	*
9138	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9139	*/
9140	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9141	&& !CPUMIsGuestInRealModeEx(pCtx))
9142	{
9143	switch (uVector)
9144	{
9145	case X86_XCPT_PF:
9146	case X86_XCPT_DF:
9147	case X86_XCPT_TS:
9148	case X86_XCPT_NP:
9149	case X86_XCPT_SS:
9150	case X86_XCPT_GP:
9151	case X86_XCPT_AC:
9152	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9153	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9154	RT_FALL_THRU();
9155	default:
9156	break;
9157	}
9158	}
9159
9160	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
9161	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9162	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9163	#endif
9164
9165	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9166
9167	/*
9168	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
9169	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9170	* interrupt handler in the (real-mode) guest.
9171	*
9172	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9173	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9174	*/
9175	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9176	{
9177	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9178	{
9179	/*
9180	* For CPUs with unrestricted guest execution enabled and with the guest
9181	* in real-mode, we must not set the deliver-error-code bit.
9182	*
9183	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9184	*/
9185	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9186	}
9187	else
9188	{
9189	PVM pVM = pVCpu->CTX_SUFF(pVM);
9190	Assert(PDMVmmDevHeapIsEnabled(pVM));
9191	Assert(pVM->hm.s.vmx.pRealModeTSS);
9192	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9193
9194	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9195	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9196	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
9197	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
9198	AssertRCReturn(rc2, rc2);
9199
9200	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9201	size_t const cbIdtEntry = sizeof(X86IDTR16);
9202	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9203	{
9204	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9205	if (uVector == X86_XCPT_DF)
9206	return VINF_EM_RESET;
9207
9208	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9209	No error codes for exceptions in real-mode. */
9210	if (uVector == X86_XCPT_GP)
9211	{
9212	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9213	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9214	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9215	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9216	HMEVENT EventXcptDf;
9217	RT_ZERO(EventXcptDf);
9218	EventXcptDf.u64IntInfo = uXcptDfInfo;
9219	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9220	}
9221
9222	/*
9223	* If we're injecting an event with no valid IDT entry, inject a #GP.
9224	* No error codes for exceptions in real-mode.
9225	*
9226	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9227	*/
9228	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9229	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9230	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9231	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9232	HMEVENT EventXcptGp;
9233	RT_ZERO(EventXcptGp);
9234	EventXcptGp.u64IntInfo = uXcptGpInfo;
9235	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9236	}
9237
9238	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9239	uint16_t uGuestIp = pCtx->ip;
9240	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9241	{
9242	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9243	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
9244	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9245	}
9246	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9247	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9248
9249	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9250	X86IDTR16 IdtEntry;
9251	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9252	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9253	AssertRCReturn(rc2, rc2);
9254
9255	/* Construct the stack frame for the interrupt/exception handler. */
9256	VBOXSTRICTRC rcStrict;
9257	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9258	if (rcStrict == VINF_SUCCESS)
9259	{
9260	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9261	if (rcStrict == VINF_SUCCESS)
9262	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9263	}
9264
9265	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
9266	if (rcStrict == VINF_SUCCESS)
9267	{
9268	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
9269	pCtx->rip = IdtEntry.offSel;
9270	pCtx->cs.Sel = IdtEntry.uSel;
9271	pCtx->cs.ValidSel = IdtEntry.uSel;
9272	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9273	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9274	&& uVector == X86_XCPT_PF)
9275	pCtx->cr2 = GCPtrFault;
9276
9277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
9278	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9279	\| HM_CHANGED_GUEST_RSP);
9280
9281	/*
9282	* If we delivered a hardware exception (other than an NMI) and if there was
9283	* block-by-STI in effect, we should clear it.
9284	*/
9285	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9286	{
9287	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9288	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9289	Log4Func(("Clearing inhibition due to STI\n"));
9290	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9291	}
9292
9293	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9294	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9295
9296	/*
9297	* The event has been truly dispatched to the guest. Mark it as no longer pending so
9298	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9299	*/
9300	pVCpu->hm.s.Event.fPending = false;
9301
9302	/*
9303	* If we eventually support nested-guest execution without unrestricted guest execution,
9304	* we should set fInterceptEvents here.
9305	*/
9306	Assert(!pVmxTransient->fIsNestedGuest);
9307
9308	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9309	if (fStepping)
9310	rcStrict = VINF_EM_DBG_STEPPED;
9311	}
9312	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9313	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9314	return rcStrict;
9315	}
9316	}
9317
9318	/*
9319	* Validate.
9320	*/
9321	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9322	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9323
9324	/*
9325	* Inject the event into the VMCS.
9326	*/
9327	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9328	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9329	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9330	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9331	AssertRCReturn(rc, rc);
9332
9333	/*
9334	* Update guest CR2 if this is a page-fault.
9335	*/
9336	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
9337	pCtx->cr2 = GCPtrFault;
9338
9339	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9340	return VINF_SUCCESS;
9341	}
9342
9343
9344	/**
9345	* Evaluates the event to be delivered to the guest and sets it as the pending
9346	* event.
9347	*
9348	* @returns Strict VBox status code (i.e. informational status codes too).
9349	* @param pVCpu The cross context virtual CPU structure.
9350	* @param pVmxTransient The VMX-transient structure.
9351	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
9352	*/
9353	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9354	{
9355	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9356	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9357	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
9358
9359	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9360	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9361	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9362	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9363	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9364
9365	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9366	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9367	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9368	Assert(!TRPMHasTrap(pVCpu));
9369	Assert(pfIntrState);
9370
9371	*pfIntrState = fIntrState;
9372
9373	/*
9374	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9375	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9376	*/
9377	/** @todo SMI. SMIs take priority over NMIs. */
9378	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9379	{
9380	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9381	if ( !pVCpu->hm.s.Event.fPending
9382	&& !fBlockNmi
9383	&& !fBlockSti
9384	&& !fBlockMovSS)
9385	{
9386	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9387	if ( fIsNestedGuest
9388	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9389	return IEMExecVmxVmexitXcptNmi(pVCpu);
9390	#endif
9391	hmR0VmxSetPendingXcptNmi(pVCpu);
9392	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9393	Log4Func(("Pending NMI\n"));
9394	}
9395	else if (!fIsNestedGuest)
9396	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9397	}
9398	/*
9399	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9400	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9401	*/
9402	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
9403	&& !pVCpu->hm.s.fSingleInstruction)
9404	{
9405	Assert(!DBGFIsStepping(pVCpu));
9406	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9407	AssertRCReturn(rc, rc);
9408	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9409	if ( !pVCpu->hm.s.Event.fPending
9410	&& !fBlockInt
9411	&& !fBlockSti
9412	&& !fBlockMovSS)
9413	{
9414	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9415	if ( fIsNestedGuest
9416	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9417	{
9418	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
9419	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9420	return rcStrict;
9421	}
9422	#endif
9423	uint8_t u8Interrupt;
9424	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9425	if (RT_SUCCESS(rc))
9426	{
9427	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9428	if ( fIsNestedGuest
9429	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9430	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9431	{
9432	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9433	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9434	return rcStrict;
9435	}
9436	#endif
9437	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9438	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9439	}
9440	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9441	{
9442	if ( !fIsNestedGuest
9443	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9444	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9445	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9446
9447	/*
9448	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9449	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9450	* need to re-set this force-flag here.
9451	*/
9452	}
9453	else
9454	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9455	}
9456	else if (!fIsNestedGuest)
9457	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9458	}
9459
9460	return VINF_SUCCESS;
9461	}
9462
9463
9464	/**
9465	* Injects any pending events into the guest if the guest is in a state to
9466	* receive them.
9467	*
9468	* @returns Strict VBox status code (i.e. informational status codes too).
9469	* @param pVCpu The cross context virtual CPU structure.
9470	* @param pVmxTransient The VMX-transient structure.
9471	* @param fIntrState The VT-x guest-interruptibility state.
9472	* @param fStepping Whether we are single-stepping the guest using the
9473	* hypervisor debugger and should return
9474	* VINF_EM_DBG_STEPPED if the event was dispatched
9475	* directly.
9476	*/
9477	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9478	{
9479	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9480	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9481
9482	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9483	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9484
9485	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9486	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9487	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9488	Assert(!TRPMHasTrap(pVCpu));
9489
9490	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9491	if (pVCpu->hm.s.Event.fPending)
9492	{
9493	/*
9494	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9495	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9496	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9497	*
9498	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9499	*/
9500	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9501	#ifdef VBOX_STRICT
9502	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9503	{
9504	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9505	Assert(!fBlockInt);
9506	Assert(!fBlockSti);
9507	Assert(!fBlockMovSS);
9508	}
9509	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9510	{
9511	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9512	Assert(!fBlockSti);
9513	Assert(!fBlockMovSS);
9514	Assert(!fBlockNmi);
9515	}
9516	#endif
9517	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9518	uIntType));
9519
9520	/*
9521	* Inject the event and get any changes to the guest-interruptibility state.
9522	*
9523	* The guest-interruptibility state may need to be updated if we inject the event
9524	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9525	*/
9526	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9527	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9528
9529	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9530	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9531	else
9532	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9533	}
9534
9535	/*
9536	* Update the guest-interruptibility state.
9537	*
9538	* This is required for the real-on-v86 software interrupt injection case above, as well as
9539	* updates to the guest state from ring-3 or IEM/REM.
9540	*/
9541	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9542	AssertRCReturn(rc, rc);
9543
9544	/*
9545	* There's no need to clear the VM-entry interruption-information field here if we're not
9546	* injecting anything. VT-x clears the valid bit on every VM-exit.
9547	*
9548	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9549	*/
9550
9551	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9552	NOREF(fBlockMovSS); NOREF(fBlockSti);
9553	return rcStrict;
9554	}
9555
9556
9557	/**
9558	* Enters the VT-x session.
9559	*
9560	* @returns VBox status code.
9561	* @param pVCpu The cross context virtual CPU structure.
9562	*/
9563	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9564	{
9565	AssertPtr(pVCpu);
9566	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9567	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9568
9569	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9570	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9571	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9572
9573	#ifdef VBOX_STRICT
9574	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9575	RTCCUINTREG uHostCr4 = ASMGetCR4();
9576	if (!(uHostCr4 & X86_CR4_VMXE))
9577	{
9578	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9579	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9580	}
9581	#endif
9582
9583	/*
9584	* Load the appropriate VMCS as the current and active one.
9585	*/
9586	PVMXVMCSINFO pVmcsInfo;
9587	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9588	if (!fInNestedGuestMode)
9589	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9590	else
9591	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9592	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9593	if (RT_SUCCESS(rc))
9594	{
9595	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9596	pVCpu->hm.s.fLeaveDone = false;
9597	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9598
9599	/*
9600	* Do the EMT scheduled L1D flush here if needed.
9601	*/
9602	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9603	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9604	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9605	hmR0MdsClear();
9606	}
9607	return rc;
9608	}
9609
9610
9611	/**
9612	* The thread-context callback (only on platforms which support it).
9613	*
9614	* @param enmEvent The thread-context event.
9615	* @param pVCpu The cross context virtual CPU structure.
9616	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9617	* @thread EMT(pVCpu)
9618	*/
9619	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9620	{
9621	AssertPtr(pVCpu);
9622	RT_NOREF1(fGlobalInit);
9623
9624	switch (enmEvent)
9625	{
9626	case RTTHREADCTXEVENT_OUT:
9627	{
9628	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9629	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9630	VMCPU_ASSERT_EMT(pVCpu);
9631
9632	/* No longjmps (logger flushes, locks) in this fragile context. */
9633	VMMRZCallRing3Disable(pVCpu);
9634	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9635
9636	/* Restore host-state (FPU, debug etc.) */
9637	if (!pVCpu->hm.s.fLeaveDone)
9638	{
9639	/*
9640	* Do -not- import the guest-state here as we might already be in the middle of importing
9641	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9642	*/
9643	hmR0VmxLeave(pVCpu, false /* fImportState */);
9644	pVCpu->hm.s.fLeaveDone = true;
9645	}
9646
9647	/* Leave HM context, takes care of local init (term). */
9648	int rc = HMR0LeaveCpu(pVCpu);
9649	AssertRC(rc);
9650
9651	/* Restore longjmp state. */
9652	VMMRZCallRing3Enable(pVCpu);
9653	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9654	break;
9655	}
9656
9657	case RTTHREADCTXEVENT_IN:
9658	{
9659	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9660	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9661	VMCPU_ASSERT_EMT(pVCpu);
9662
9663	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9664	VMMRZCallRing3Disable(pVCpu);
9665	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9666
9667	/* Initialize the bare minimum state required for HM. This takes care of
9668	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9669	int rc = hmR0EnterCpu(pVCpu);
9670	AssertRC(rc);
9671	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9672	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9673
9674	/* Load the active VMCS as the current one. */
9675	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9676	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9677	AssertRC(rc);
9678	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9679	pVCpu->hm.s.fLeaveDone = false;
9680
9681	/* Do the EMT scheduled L1D flush if needed. */
9682	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9683	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9684
9685	/* Restore longjmp state. */
9686	VMMRZCallRing3Enable(pVCpu);
9687	break;
9688	}
9689
9690	default:
9691	break;
9692	}
9693	}
9694
9695
9696	/**
9697	* Exports the host state into the VMCS host-state area.
9698	* Sets up the VM-exit MSR-load area.
9699	*
9700	* The CPU state will be loaded from these fields on every successful VM-exit.
9701	*
9702	* @returns VBox status code.
9703	* @param pVCpu The cross context virtual CPU structure.
9704	*
9705	* @remarks No-long-jump zone!!!
9706	*/
9707	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9708	{
9709	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9710
9711	int rc = VINF_SUCCESS;
9712	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9713	{
9714	rc = hmR0VmxExportHostControlRegs();
9715	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9716
9717	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9718	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9719
9720	rc = hmR0VmxExportHostMsrs(pVCpu);
9721	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9722
9723	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9724	}
9725	return rc;
9726	}
9727
9728
9729	/**
9730	* Saves the host state in the VMCS host-state.
9731	*
9732	* @returns VBox status code.
9733	* @param pVCpu The cross context virtual CPU structure.
9734	*
9735	* @remarks No-long-jump zone!!!
9736	*/
9737	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9738	{
9739	AssertPtr(pVCpu);
9740	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9741
9742	/*
9743	* Export the host state here while entering HM context.
9744	* When thread-context hooks are used, we might get preempted and have to re-save the host
9745	* state but most of the time we won't be, so do it here before we disable interrupts.
9746	*/
9747	return hmR0VmxExportHostState(pVCpu);
9748	}
9749
9750
9751	/**
9752	* Exports the guest state into the VMCS guest-state area.
9753	*
9754	* The will typically be done before VM-entry when the guest-CPU state and the
9755	* VMCS state may potentially be out of sync.
9756	*
9757	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9758	* VM-entry controls.
9759	* Sets up the appropriate VMX non-root function to execute guest code based on
9760	* the guest CPU mode.
9761	*
9762	* @returns VBox strict status code.
9763	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9764	* without unrestricted guest execution and the VMMDev is not presently
9765	* mapped (e.g. EFI32).
9766	*
9767	* @param pVCpu The cross context virtual CPU structure.
9768	* @param pVmxTransient The VMX-transient structure.
9769	*
9770	* @remarks No-long-jump zone!!!
9771	*/
9772	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9773	{
9774	AssertPtr(pVCpu);
9775	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9776	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9777
9778	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9779
9780	/*
9781	* Determine real-on-v86 mode.
9782	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9783	*/
9784	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9785	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9786	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9787	pVmcsInfo->RealMode. fRealOnV86Active = false;
9788	else
9789	{
9790	Assert(!pVmxTransient->fIsNestedGuest);
9791	pVmcsInfo->RealMode.fRealOnV86Active = true;
9792	}
9793
9794	/*
9795	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9796	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9797	*/
9798	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9799	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9800	* be a need to evaluate this everytime since I'm pretty sure we intercept
9801	* all guest paging mode changes. */
9802	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9803	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9804
9805	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9806	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9807
9808	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9809	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9810
9811	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9812	if (rcStrict == VINF_SUCCESS)
9813	{ /* likely */ }
9814	else
9815	{
9816	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9817	return rcStrict;
9818	}
9819
9820	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9821	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9822
9823	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9824	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9825
9826	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9827	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9828
9829	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9830	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9831
9832	rc = hmR0VmxExportGuestRip(pVCpu);
9833	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9834	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9835	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9836
9837	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9838	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9839
9840	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9841	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9842	\| HM_CHANGED_GUEST_CR2
9843	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9844	\| HM_CHANGED_GUEST_X87
9845	\| HM_CHANGED_GUEST_SSE_AVX
9846	\| HM_CHANGED_GUEST_OTHER_XSAVE
9847	\| HM_CHANGED_GUEST_XCRx
9848	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9849	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9850	\| HM_CHANGED_GUEST_TSC_AUX
9851	\| HM_CHANGED_GUEST_OTHER_MSRS
9852	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9853
9854	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9855	return rc;
9856	}
9857
9858
9859	/**
9860	* Exports the state shared between the host and guest into the VMCS.
9861	*
9862	* @param pVCpu The cross context virtual CPU structure.
9863	* @param pVmxTransient The VMX-transient structure.
9864	*
9865	* @remarks No-long-jump zone!!!
9866	*/
9867	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9868	{
9869	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9870	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9871
9872	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9873	{
9874	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9875	AssertRC(rc);
9876	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9877
9878	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9879	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9880	{
9881	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9882	AssertRC(rc);
9883	}
9884	}
9885
9886	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9887	{
9888	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9889	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9890	}
9891
9892	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9893	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9894	}
9895
9896
9897	/**
9898	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9899	*
9900	* @returns Strict VBox status code (i.e. informational status codes too).
9901	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9902	* without unrestricted guest execution and the VMMDev is not presently
9903	* mapped (e.g. EFI32).
9904	*
9905	* @param pVCpu The cross context virtual CPU structure.
9906	* @param pVmxTransient The VMX-transient structure.
9907	*
9908	* @remarks No-long-jump zone!!!
9909	*/
9910	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9911	{
9912	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9913	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9914	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9915
9916	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9917	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9918	#endif
9919
9920	/*
9921	* For many exits it's only RIP that changes and hence try to export it first
9922	* without going through a lot of change flag checks.
9923	*/
9924	VBOXSTRICTRC rcStrict;
9925	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9926	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9927	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9928	{
9929	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9930	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9931	{ /* likely */}
9932	else
9933	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9935	}
9936	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9937	{
9938	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9939	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9940	{ /* likely */}
9941	else
9942	{
9943	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9944	VBOXSTRICTRC_VAL(rcStrict)));
9945	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9946	return rcStrict;
9947	}
9948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9949	}
9950	else
9951	rcStrict = VINF_SUCCESS;
9952
9953	#ifdef VBOX_STRICT
9954	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9955	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9956	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9957	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9958	("fCtxChanged=%#RX64\n", fCtxChanged));
9959	#endif
9960	return rcStrict;
9961	}
9962
9963
9964	/**
9965	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9966	* and update error record fields accordingly.
9967	*
9968	* @returns VMX_IGS_* error codes.
9969	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9970	* wrong with the guest state.
9971	*
9972	* @param pVCpu The cross context virtual CPU structure.
9973	* @param pVmcsInfo The VMCS info. object.
9974	*
9975	* @remarks This function assumes our cache of the VMCS controls
9976	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9977	*/
9978	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9979	{
9980	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9981	#define HMVMX_CHECK_BREAK(expr, err) do { \
9982	if (!(expr)) { uError = (err); break; } \
9983	} while (0)
9984
9985	int rc;
9986	PVM pVM = pVCpu->CTX_SUFF(pVM);
9987	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9988	uint32_t uError = VMX_IGS_ERROR;
9989	uint32_t u32Val;
9990	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9991
9992	do
9993	{
9994	/*
9995	* CR0.
9996	*/
9997	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9998	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9999	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10000	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10001	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10002	if (fUnrestrictedGuest)
10003	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10004
10005	uint32_t u32GuestCr0;
10006	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10007	AssertRCBreak(rc);
10008	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10009	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10010	if ( !fUnrestrictedGuest
10011	&& (u32GuestCr0 & X86_CR0_PG)
10012	&& !(u32GuestCr0 & X86_CR0_PE))
10013	{
10014	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10015	}
10016
10017	/*
10018	* CR4.
10019	*/
10020	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10021	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10022	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10023
10024	uint32_t u32GuestCr4;
10025	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10026	AssertRCBreak(rc);
10027	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10028	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10029
10030	/*
10031	* IA32_DEBUGCTL MSR.
10032	*/
10033	uint64_t u64Val;
10034	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10035	AssertRCBreak(rc);
10036	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10037	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10038	{
10039	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10040	}
10041	uint64_t u64DebugCtlMsr = u64Val;
10042
10043	#ifdef VBOX_STRICT
10044	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10045	AssertRCBreak(rc);
10046	Assert(u32Val == pVmcsInfo->u32EntryCtls);
10047	#endif
10048	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10049
10050	/*
10051	* RIP and RFLAGS.
10052	*/
10053	uint32_t u32Eflags;
10054	#if HC_ARCH_BITS == 64
10055	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10056	AssertRCBreak(rc);
10057	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10058	if ( !fLongModeGuest
10059	\|\| !pCtx->cs.Attr.n.u1Long)
10060	{
10061	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10062	}
10063	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10064	* must be identical if the "IA-32e mode guest" VM-entry
10065	* control is 1 and CS.L is 1. No check applies if the
10066	* CPU supports 64 linear-address bits. */
10067
10068	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10069	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10070	AssertRCBreak(rc);
10071	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10072	VMX_IGS_RFLAGS_RESERVED);
10073	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10074	u32Eflags = u64Val;
10075	#else
10076	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10077	AssertRCBreak(rc);
10078	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10079	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10080	#endif
10081
10082	if ( fLongModeGuest
10083	\|\| ( fUnrestrictedGuest
10084	&& !(u32GuestCr0 & X86_CR0_PE)))
10085	{
10086	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10087	}
10088
10089	uint32_t u32EntryInfo;
10090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10091	AssertRCBreak(rc);
10092	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
10093	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10094
10095	/*
10096	* 64-bit checks.
10097	*/
10098	#if HC_ARCH_BITS == 64
10099	if (fLongModeGuest)
10100	{
10101	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10102	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10103	}
10104
10105	if ( !fLongModeGuest
10106	&& (u32GuestCr4 & X86_CR4_PCIDE))
10107	{
10108	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10109	}
10110
10111	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10112	* 51:32 beyond the processor's physical-address width are 0. */
10113
10114	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10115	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10116	{
10117	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10118	}
10119
10120	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10121	AssertRCBreak(rc);
10122	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10123
10124	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10125	AssertRCBreak(rc);
10126	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10127	#endif
10128
10129	/*
10130	* PERF_GLOBAL MSR.
10131	*/
10132	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10133	{
10134	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10135	AssertRCBreak(rc);
10136	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10137	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10138	}
10139
10140	/*
10141	* PAT MSR.
10142	*/
10143	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10144	{
10145	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10146	AssertRCBreak(rc);
10147	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10148	for (unsigned i = 0; i < 8; i++)
10149	{
10150	uint8_t u8Val = (u64Val & 0xff);
10151	if ( u8Val != 0 /* UC */
10152	&& u8Val != 1 /* WC */
10153	&& u8Val != 4 /* WT */
10154	&& u8Val != 5 /* WP */
10155	&& u8Val != 6 /* WB */
10156	&& u8Val != 7 /* UC- */)
10157	{
10158	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10159	}
10160	u64Val >>= 8;
10161	}
10162	}
10163
10164	/*
10165	* EFER MSR.
10166	*/
10167	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10168	{
10169	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10170	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10171	AssertRCBreak(rc);
10172	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10173	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10174	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10175	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10176	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10177	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10178	* iemVmxVmentryCheckGuestState(). */
10179	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10180	\|\| !(u32GuestCr0 & X86_CR0_PG)
10181	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10182	VMX_IGS_EFER_LMA_LME_MISMATCH);
10183	}
10184
10185	/*
10186	* Segment registers.
10187	*/
10188	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10189	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10190	if (!(u32Eflags & X86_EFL_VM))
10191	{
10192	/* CS */
10193	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10194	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10195	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10196	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10197	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10198	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10199	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10200	/* CS cannot be loaded with NULL in protected mode. */
10201	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10202	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10203	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10204	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10205	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10206	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10207	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10208	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10209	else
10210	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10211
10212	/* SS */
10213	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10214	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10215	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10216	if ( !(pCtx->cr0 & X86_CR0_PE)
10217	\|\| pCtx->cs.Attr.n.u4Type == 3)
10218	{
10219	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10220	}
10221	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10222	{
10223	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10224	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10225	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10226	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10227	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10228	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10229	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10230	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10231	}
10232
10233	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10234	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10235	{
10236	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10237	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10238	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10239	\|\| pCtx->ds.Attr.n.u4Type > 11
10240	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10241	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10242	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10243	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10244	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10245	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10246	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10247	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10248	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10249	}
10250	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10251	{
10252	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10253	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10254	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10255	\|\| pCtx->es.Attr.n.u4Type > 11
10256	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10257	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10258	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10259	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10260	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10261	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10262	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10263	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10264	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10265	}
10266	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10267	{
10268	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10269	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10270	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10271	\|\| pCtx->fs.Attr.n.u4Type > 11
10272	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10273	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10274	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10275	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10276	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10277	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10278	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10279	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10280	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10281	}
10282	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10283	{
10284	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10285	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10286	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10287	\|\| pCtx->gs.Attr.n.u4Type > 11
10288	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10289	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10290	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10291	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10292	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10293	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10294	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10295	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10296	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10297	}
10298	/* 64-bit capable CPUs. */
10299	#if HC_ARCH_BITS == 64
10300	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10301	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10302	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10303	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10304	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10305	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10306	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10307	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10308	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10309	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10310	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10311	#endif
10312	}
10313	else
10314	{
10315	/* V86 mode checks. */
10316	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10317	if (pVmcsInfo->RealMode.fRealOnV86Active)
10318	{
10319	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10320	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10321	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10322	}
10323	else
10324	{
10325	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10326	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10327	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10328	}
10329
10330	/* CS */
10331	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10332	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10333	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10334	/* SS */
10335	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10336	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10337	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10338	/* DS */
10339	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10340	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10341	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10342	/* ES */
10343	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10344	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10345	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10346	/* FS */
10347	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10348	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10349	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10350	/* GS */
10351	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10352	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10353	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10354	/* 64-bit capable CPUs. */
10355	#if HC_ARCH_BITS == 64
10356	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10357	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10358	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10359	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10360	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10361	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10362	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10363	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10364	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10365	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10366	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10367	#endif
10368	}
10369
10370	/*
10371	* TR.
10372	*/
10373	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10374	/* 64-bit capable CPUs. */
10375	#if HC_ARCH_BITS == 64
10376	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10377	#endif
10378	if (fLongModeGuest)
10379	{
10380	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10381	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10382	}
10383	else
10384	{
10385	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10386	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10387	VMX_IGS_TR_ATTR_TYPE_INVALID);
10388	}
10389	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10390	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10391	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10392	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10393	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10394	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10395	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10396	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10397
10398	/*
10399	* GDTR and IDTR.
10400	*/
10401	#if HC_ARCH_BITS == 64
10402	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10403	AssertRCBreak(rc);
10404	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10405
10406	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10407	AssertRCBreak(rc);
10408	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10409	#endif
10410
10411	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10412	AssertRCBreak(rc);
10413	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10414
10415	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10416	AssertRCBreak(rc);
10417	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10418
10419	/*
10420	* Guest Non-Register State.
10421	*/
10422	/* Activity State. */
10423	uint32_t u32ActivityState;
10424	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10425	AssertRCBreak(rc);
10426	HMVMX_CHECK_BREAK( !u32ActivityState
10427	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10428	VMX_IGS_ACTIVITY_STATE_INVALID);
10429	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10430	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10431	uint32_t u32IntrState;
10432	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10433	AssertRCBreak(rc);
10434	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10435	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10436	{
10437	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10438	}
10439
10440	/** @todo Activity state and injecting interrupts. Left as a todo since we
10441	* currently don't use activity states but ACTIVE. */
10442
10443	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10444	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10445
10446	/* Guest interruptibility-state. */
10447	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10448	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10449	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10450	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10451	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10452	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10453	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10454	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
10455	{
10456	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10457	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10458	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10459	}
10460	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10461	{
10462	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10463	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10464	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10465	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10466	}
10467	/** @todo Assumes the processor is not in SMM. */
10468	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10469	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10470	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10471	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10472	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10473	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10474	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10475	{
10476	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10477	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10478	}
10479
10480	/* Pending debug exceptions. */
10481	#if HC_ARCH_BITS == 64
10482	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10483	AssertRCBreak(rc);
10484	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10485	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10486	u32Val = u64Val; /* For pending debug exceptions checks below. */
10487	#else
10488	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10489	AssertRCBreak(rc);
10490	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10491	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10492	#endif
10493
10494	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10495	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10496	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10497	{
10498	if ( (u32Eflags & X86_EFL_TF)
10499	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10500	{
10501	/* Bit 14 is PendingDebug.BS. */
10502	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10503	}
10504	if ( !(u32Eflags & X86_EFL_TF)
10505	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10506	{
10507	/* Bit 14 is PendingDebug.BS. */
10508	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10509	}
10510	}
10511
10512	/* VMCS link pointer. */
10513	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10514	AssertRCBreak(rc);
10515	if (u64Val != UINT64_C(0xffffffffffffffff))
10516	{
10517	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10518	/** @todo Bits beyond the processor's physical-address width MBZ. */
10519	/** @todo SMM checks. */
10520	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10521	Assert(pVmcsInfo->pvShadowVmcs);
10522	VMXVMCSREVID VmcsRevId;
10523	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10524	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10525	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10526	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10527	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10528	}
10529
10530	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10531	* not using nested paging? */
10532	if ( pVM->hm.s.fNestedPaging
10533	&& !fLongModeGuest
10534	&& CPUMIsGuestInPAEModeEx(pCtx))
10535	{
10536	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10537	AssertRCBreak(rc);
10538	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10539
10540	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10541	AssertRCBreak(rc);
10542	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10543
10544	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10545	AssertRCBreak(rc);
10546	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10547
10548	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10549	AssertRCBreak(rc);
10550	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10551	}
10552
10553	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10554	if (uError == VMX_IGS_ERROR)
10555	uError = VMX_IGS_REASON_NOT_FOUND;
10556	} while (0);
10557
10558	pVCpu->hm.s.u32HMError = uError;
10559	return uError;
10560
10561	#undef HMVMX_ERROR_BREAK
10562	#undef HMVMX_CHECK_BREAK
10563	}
10564
10565
10566	/**
10567	* Map the APIC-access page for virtualizing APIC accesses.
10568	*
10569	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10570	* this not done as part of exporting guest state, see @bugref{8721}.
10571	*
10572	* @returns VBox status code.
10573	* @param pVCpu The cross context virtual CPU structure.
10574	*/
10575	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10576	{
10577	PVM pVM = pVCpu->CTX_SUFF(pVM);
10578	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10579
10580	Assert(PDMHasApic(pVM));
10581	Assert(u64MsrApicBase);
10582
10583	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10584	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10585
10586	/* Unalias the existing mapping. */
10587	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10588	AssertRCReturn(rc, rc);
10589
10590	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10591	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10592	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10593	AssertRCReturn(rc, rc);
10594
10595	/* Update the per-VCPU cache of the APIC base MSR. */
10596	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10597	return VINF_SUCCESS;
10598	}
10599
10600
10601	/**
10602	* Worker function passed to RTMpOnSpecific() that is to be called on the target
10603	* CPU.
10604	*
10605	* @param idCpu The ID for the CPU the function is called on.
10606	* @param pvUser1 Null, not used.
10607	* @param pvUser2 Null, not used.
10608	*/
10609	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
10610	{
10611	RT_NOREF3(idCpu, pvUser1, pvUser2);
10612	VMXDispatchHostNmi();
10613	}
10614
10615
10616	/**
10617	* Dispatching an NMI on the host CPU that received it.
10618	*
10619	* @returns VBox status code.
10620	* @param pVCpu The cross context virtual CPU structure.
10621	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
10622	* executing when receiving the host NMI in VMX non-root
10623	* operation.
10624	*/
10625	static int hmR0VmxExitHostNmi(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10626	{
10627	RTCPUID const idCpu = pVmcsInfo->idHostCpu;
10628
10629	/*
10630	* We don't want to delay dispatching the NMI any more than we have to. However,
10631	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
10632	* after executing guest or nested-guest code for the following reasons:
10633	*
10634	* - We would need to perform VMREADs with interrupts disabled and is orders of
10635	* magnitude worse with nested virtualization.
10636	*
10637	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
10638	* longer period of time just for handling an edge case like host NMIs.
10639	*
10640	* Let's cover the most likely scenario first. Check if we are on the target CPU
10641	* and dispatch the NMI right away. This should be much faster than calling into
10642	* RTMpOnSpecific() machinery.
10643	*/
10644	bool fDispatched = false;
10645	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10646	if (idCpu == RTMpCpuId())
10647	{
10648	VMXDispatchHostNmi();
10649	fDispatched = true;
10650	}
10651	ASMSetFlags(fEFlags);
10652	if (fDispatched)
10653	{
10654	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10655	return VINF_SUCCESS;
10656	}
10657
10658	/*
10659	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
10660	* there should be no race or recursion even if we are unlucky enough to be preempted
10661	* (to the target CPU) without dispatching the host NMI above.
10662	*/
10663	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
10664	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
10665	}
10666
10667
10668	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10669	/**
10670	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10671	* nested-guest using hardware-assisted VMX.
10672	*
10673	* @param pVCpu The cross context virtual CPU structure.
10674	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10675	* @param pVmcsInfoGst The guest VMCS info. object.
10676	*/
10677	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10678	{
10679	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10680	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10681	Assert(pu64MsrBitmap);
10682
10683	/*
10684	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10685	* MSR that is intercepted by the guest is also intercepted while executing the
10686	* nested-guest using hardware-assisted VMX.
10687	*
10688	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10689	* nested-guest VM-exit even if the outer guest is not intercepting some
10690	* MSRs. We cannot assume the caller has initialized the nested-guest
10691	* MSR bitmap in this case.
10692	*
10693	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10694	* each VM-entry, hence initializing it once per-VM while setting up the
10695	* nested-guest VMCS is not sufficient.
10696	*/
10697	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10698	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10699	{
10700	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10701	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10702	Assert(pu64MsrBitmapNstGst);
10703	Assert(pu64MsrBitmapGst);
10704
10705	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10706	for (uint32_t i = 0; i < cFrags; i++)
10707	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10708	}
10709	else
10710	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10711	}
10712
10713
10714	/**
10715	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10716	* hardware-assisted VMX execution of the nested-guest.
10717	*
10718	* For a guest, we don't modify these controls once we set up the VMCS and hence
10719	* this function is never called.
10720	*
10721	* For nested-guests since the guest hypervisor provides these controls on every
10722	* nested-guest VM-entry and could potentially change them everytime we need to
10723	* merge them before every nested-guest VM-entry.
10724	*
10725	* @returns VBox status code.
10726	* @param pVCpu The cross context virtual CPU structure.
10727	*/
10728	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10729	{
10730	PVM pVM = pVCpu->CTX_SUFF(pVM);
10731	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10732	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10733	Assert(pVmcsNstGst);
10734
10735	/*
10736	* Merge the controls with the requirements of the guest VMCS.
10737	*
10738	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10739	* VMCS with the features supported by the physical CPU as it's already done by the
10740	* VMLAUNCH/VMRESUME instruction emulation.
10741	*
10742	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10743	* derived from the VMX features supported by the physical CPU.
10744	*/
10745
10746	/* Pin-based VM-execution controls. */
10747	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10748
10749	/* Processor-based VM-execution controls. */
10750	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10751	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10752	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10753	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10754	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10755
10756	/* Secondary processor-based VM-execution controls. */
10757	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10758	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10759	\| VMX_PROC_CTLS2_INVPCID
10760	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10761	\| VMX_PROC_CTLS2_RDTSCP
10762	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10763	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10764	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10765	\| VMX_PROC_CTLS2_VMFUNC));
10766
10767	/*
10768	* VM-entry controls:
10769	* These controls contains state that depends on the nested-guest state (primarily
10770	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10771	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10772	* properly continue executing the nested-guest if the EFER MSR changes but does not
10773	* cause a nested-guest VM-exits.
10774	*
10775	* VM-exit controls:
10776	* These controls specify the host state on return. We cannot use the controls from
10777	* the guest hypervisor state as is as it would contain the guest state rather than
10778	* the host state. Since the host state is subject to change (e.g. preemption, trips
10779	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10780	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10781	*
10782	* VM-entry MSR-load:
10783	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10784	* context by the VMLAUNCH/VMRESUME instruction emulation.
10785	*
10786	* VM-exit MSR-store:
10787	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10788	* back into the VM-exit MSR-store area.
10789	*
10790	* VM-exit MSR-load areas:
10791	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10792	* can entirely ignore what the guest hypervisor wants to load here.
10793	*/
10794
10795	/*
10796	* Exception bitmap.
10797	*
10798	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10799	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10800	* code more flexible if intercepting exceptions become more dynamic in the future we do
10801	* it as part of exporting the nested-guest state.
10802	*/
10803	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10804
10805	/*
10806	* CR0/CR4 guest/host mask.
10807	*
10808	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10809	* cause VM-exits, so we need to merge them here.
10810	*/
10811	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10812	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10813
10814	/*
10815	* Page-fault error-code mask and match.
10816	*
10817	* Although we require unrestricted guest execution (and thereby nested-paging) for
10818	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10819	* normally intercept #PFs, it might intercept them for debugging purposes.
10820	*
10821	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10822	* If the outer guest is intercepting #PFs we must intercept all #PFs.
10823	*/
10824	uint32_t u32XcptPFMask;
10825	uint32_t u32XcptPFMatch;
10826	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10827	{
10828	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10829	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10830	}
10831	else
10832	{
10833	u32XcptPFMask = 0;
10834	u32XcptPFMatch = 0;
10835	}
10836
10837	/*
10838	* Pause-Loop exiting.
10839	*/
10840	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10841	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10842
10843	/*
10844	* I/O Bitmap.
10845	*
10846	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10847	* intercept all I/O port accesses.
10848	*/
10849	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10850
10851	/*
10852	* VMCS shadowing.
10853	*
10854	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10855	* enabled while executing the nested-guest.
10856	*/
10857	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10858
10859	/*
10860	* APIC-access page.
10861	*
10862	* The APIC-access page address has already been initialized while setting up the
10863	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it should
10864	* not be of any consequence to the host or to the guest for that matter, but we only
10865	* accept valid addresses verified by the VMLAUNCH/VMRESUME instruction emulation to keep
10866	* it simple.
10867	*/
10868
10869	/*
10870	* Virtual-APIC page and TPR threshold.
10871	*
10872	* The virtual-APIC page has already been allocated (by CPUM during VM startup) and cached
10873	* from guest memory as part of VMLAUNCH/VMRESUME instruction emulation. The host physical
10874	* address has also been updated in the nested-guest VMCS.
10875	*/
10876	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10877	RTHCPHYS HCPhysVirtApic;
10878	uint32_t u32TprThreshold;
10879	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10880	{
10881	HCPhysVirtApic = pVmcsInfoNstGst->HCPhysVirtApic;
10882	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10883	}
10884	else
10885	{
10886	HCPhysVirtApic = 0;
10887	u32TprThreshold = 0;
10888
10889	/*
10890	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10891	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10892	* be taken care of by EPT/shadow paging.
10893	*/
10894	if (pVM->hm.s.fAllow64BitGuests)
10895	{
10896	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10897	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10898	}
10899	}
10900
10901	/*
10902	* Validate basic assumptions.
10903	*/
10904	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10905	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10906	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10907
10908	/*
10909	* Commit it to the nested-guest VMCS.
10910	*/
10911	int rc = VINF_SUCCESS;
10912	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10913	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10914	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10915	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10916	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10917	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10918	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10919	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10920	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10921	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10922	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10923	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10924	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10925	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10926	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10927	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10928	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10929	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10930	{
10931	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10932	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10933	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10934	}
10935	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10936	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVmcsInfoNstGst->HCPhysVirtApic);
10937	AssertRCReturn(rc, rc);
10938
10939	/*
10940	* Update the nested-guest VMCS cache.
10941	*/
10942	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10943	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10944	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10945	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10946	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10947	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10948	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10949	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10950	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10951
10952	/*
10953	* MSR bitmap.
10954	*
10955	* The MSR bitmap address has already been initialized while setting up the nested-guest
10956	* VMCS, here we need to merge the MSR bitmaps.
10957	*/
10958	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10959	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10960
10961	return VINF_SUCCESS;
10962	}
10963	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10964
10965
10966	/**
10967	* Does the preparations before executing guest code in VT-x.
10968	*
10969	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10970	* recompiler/IEM. We must be cautious what we do here regarding committing
10971	* guest-state information into the VMCS assuming we assuredly execute the
10972	* guest in VT-x mode.
10973	*
10974	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10975	* the common-state (TRPM/forceflags), we must undo those changes so that the
10976	* recompiler/IEM can (and should) use them when it resumes guest execution.
10977	* Otherwise such operations must be done when we can no longer exit to ring-3.
10978	*
10979	* @returns Strict VBox status code (i.e. informational status codes too).
10980	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10981	* have been disabled.
10982	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10983	* pending events).
10984	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10985	* double-fault into the guest.
10986	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10987	* dispatched directly.
10988	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10989	*
10990	* @param pVCpu The cross context virtual CPU structure.
10991	* @param pVmxTransient The VMX-transient structure.
10992	* @param fStepping Whether we are single-stepping the guest in the
10993	* hypervisor debugger. Makes us ignore some of the reasons
10994	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10995	* if event dispatching took place.
10996	*/
10997	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10998	{
10999	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11000
11001	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
11002	if (pVmxTransient->fIsNestedGuest)
11003	{
11004	RT_NOREF2(pVCpu, fStepping);
11005	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
11006	return VINF_EM_RESCHEDULE_REM;
11007	}
11008	#endif
11009
11010	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
11011	PGMRZDynMapFlushAutoSet(pVCpu);
11012	#endif
11013
11014	/*
11015	* Check and process force flag actions, some of which might require us to go back to ring-3.
11016	*/
11017	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11018	if (rcStrict == VINF_SUCCESS)
11019	{ /* FFs don't get set all the time. */ }
11020	else
11021	return rcStrict;
11022
11023	/*
11024	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
11025	*/
11026	/** @todo Doing this from ring-3 after VM setup phase causes a
11027	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
11028	* idea why atm. */
11029	PVM pVM = pVCpu->CTX_SUFF(pVM);
11030	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11031	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11032	&& PDMHasApic(pVM))
11033	{
11034	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11035	AssertRCReturn(rc, rc);
11036	}
11037
11038	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11039	/*
11040	* Merge guest VMCS controls with the nested-guest VMCS controls.
11041	*
11042	* Even if we have not executed the guest prior to this (e.g. when resuming from a
11043	* saved state), we should be okay with merging controls as we initialize the
11044	* guest VMCS controls as part of VM setup phase.
11045	*/
11046	if ( pVmxTransient->fIsNestedGuest
11047	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
11048	{
11049	int rc = hmR0VmxMergeVmcsNested(pVCpu);
11050	AssertRCReturn(rc, rc);
11051	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11052	}
11053	#endif
11054
11055	/*
11056	* Evaluate events to be injected into the guest.
11057	*
11058	* Events in TRPM can be injected without inspecting the guest state.
11059	* If any new events (interrupts/NMI) are pending currently, we try to set up the
11060	* guest to cause a VM-exit the next time they are ready to receive the event.
11061	*
11062	* With nested-guests, evaluating pending events may cause VM-exits.
11063	*/
11064	if (TRPMHasTrap(pVCpu))
11065	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11066
11067	uint32_t fIntrState;
11068	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11069
11070	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11071	/*
11072	* While evaluating pending events if something failed (unlikely) or if we were
11073	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11074	*/
11075	if (rcStrict != VINF_SUCCESS)
11076	return rcStrict;
11077	if ( pVmxTransient->fIsNestedGuest
11078	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11079	{
11080	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11081	return VINF_VMX_VMEXIT;
11082	}
11083	#else
11084	Assert(rcStrict == VINF_SUCCESS);
11085	#endif
11086
11087	/*
11088	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11089	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11090	* also result in triple-faulting the VM.
11091	*
11092	* With nested-guests, the above does not apply since unrestricted guest execution is a
11093	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
11094	*/
11095	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11096	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11097	{ /* likely */ }
11098	else
11099	{
11100	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11101	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11102	return rcStrict;
11103	}
11104
11105	/*
11106	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
11107	* import CR3 themselves. We will need to update them here, as even as late as the above
11108	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11109	* the below force flags to be set.
11110	*/
11111	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11112	{
11113	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11114	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11115	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
11116	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11117	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11118	}
11119	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11120	{
11121	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11122	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11123	}
11124
11125	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11126	/* Paranoia. */
11127	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11128	#endif
11129
11130	/*
11131	* No longjmps to ring-3 from this point on!!!
11132	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11133	* This also disables flushing of the R0-logger instance (if any).
11134	*/
11135	VMMRZCallRing3Disable(pVCpu);
11136
11137	/*
11138	* Export the guest state bits.
11139	*
11140	* We cannot perform longjmps while loading the guest state because we do not preserve the
11141	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
11142	* CPU migration.
11143	*
11144	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11145	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
11146	*/
11147	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11148	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11149	{ /* likely */ }
11150	else
11151	{
11152	VMMRZCallRing3Enable(pVCpu);
11153	return rcStrict;
11154	}
11155
11156	/*
11157	* We disable interrupts so that we don't miss any interrupts that would flag preemption
11158	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11159	* preemption disabled for a while. Since this is purely to aid the
11160	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11161	* disable interrupt on NT.
11162	*
11163	* We need to check for force-flags that could've possible been altered since we last
11164	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11165	* see @bugref{6398}).
11166	*
11167	* We also check a couple of other force-flags as a last opportunity to get the EMT back
11168	* to ring-3 before executing guest code.
11169	*/
11170	pVmxTransient->fEFlags = ASMIntDisableFlags();
11171
11172	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
11173	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11174	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11175	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
11176	{
11177	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11178	{
11179	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11180	/*
11181	* If we are executing a nested-guest make sure that we should intercept subsequent
11182	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
11183	* the VM-exit instruction emulation happy.
11184	*/
11185	if (pVmxTransient->fIsNestedGuest)
11186	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
11187	#endif
11188
11189	/*
11190	* We've injected any pending events. This is really the point of no return (to ring-3).
11191	*
11192	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
11193	* returns from this function, so do -not- enable them here.
11194	*/
11195	pVCpu->hm.s.Event.fPending = false;
11196	return VINF_SUCCESS;
11197	}
11198
11199	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11200	rcStrict = VINF_EM_RAW_INTERRUPT;
11201	}
11202	else
11203	{
11204	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11205	rcStrict = VINF_EM_RAW_TO_R3;
11206	}
11207
11208	ASMSetFlags(pVmxTransient->fEFlags);
11209	VMMRZCallRing3Enable(pVCpu);
11210
11211	return rcStrict;
11212	}
11213
11214
11215	/**
11216	* Final preparations before executing guest code using hardware-assisted VMX.
11217	*
11218	* We can no longer get preempted to a different host CPU and there are no returns
11219	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11220	* failures), this function is not intended to fail sans unrecoverable hardware
11221	* errors.
11222	*
11223	* @param pVCpu The cross context virtual CPU structure.
11224	* @param pVmxTransient The VMX-transient structure.
11225	*
11226	* @remarks Called with preemption disabled.
11227	* @remarks No-long-jump zone!!!
11228	*/
11229	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11230	{
11231	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11232	Assert(VMMR0IsLogFlushDisabled(pVCpu));
11233	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11234	Assert(!pVCpu->hm.s.Event.fPending);
11235
11236	/*
11237	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11238	*/
11239	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11240	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11241
11242	PVM pVM = pVCpu->CTX_SUFF(pVM);
11243	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11244	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11245	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11246
11247	if (!CPUMIsGuestFPUStateActive(pVCpu))
11248	{
11249	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11250	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11251	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
11252	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11253	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11254	}
11255
11256	/*
11257	* Re-export the host state bits as we may've been preempted (only happens when
11258	* thread-context hooks are used or when the VM start function changes) or if
11259	* the host CR0 is modified while loading the guest FPU state above.
11260	*
11261	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11262	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
11263	* see @bugref{8432}.
11264	*
11265	* This may also happen when switching to/from a nested-guest VMCS without leaving
11266	* ring-0.
11267	*/
11268	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11269	{
11270	hmR0VmxExportHostState(pVCpu);
11271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
11272	}
11273	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11274
11275	/*
11276	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
11277	*/
11278	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11279	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11280	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11281
11282	/*
11283	* Store status of the shared guest/host debug state at the time of VM-entry.
11284	*/
11285	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11286	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11287	{
11288	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11289	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11290	}
11291	else
11292	#endif
11293	{
11294	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11295	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11296	}
11297
11298	/*
11299	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11300	* more than one conditional check. The post-run side of our code shall determine
11301	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11302	*/
11303	if (pVmcsInfo->pbVirtApic)
11304	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11305
11306	/*
11307	* Update the host MSRs values in the VM-exit MSR-load area.
11308	*/
11309	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11310	{
11311	if (pVmcsInfo->cExitMsrLoad > 0)
11312	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11313	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11314	}
11315
11316	/*
11317	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11318	* VMX-preemption timer based on the next virtual sync clock deadline.
11319	*/
11320	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11321	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
11322	{
11323	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11324	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11325	}
11326
11327	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
11328	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
11329	if (!fIsRdtscIntercepted)
11330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
11331	else
11332	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
11333
11334	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11335	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11336	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11337	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11338	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
11339
11340	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11341
11342	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11343	as we're about to start executing the guest . */
11344
11345	/*
11346	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11347	*
11348	* This is done this late as updating the TSC offsetting/preemption timer above
11349	* figures out if we can skip intercepting RDTSCP by calculating the number of
11350	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
11351	*/
11352	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11353	&& !fIsRdtscIntercepted)
11354	{
11355	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11356
11357	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11358	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11359	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11360	true /* fSetReadWrite /, true / fUpdateHostMsr */);
11361	AssertRC(rc);
11362	Assert(!pVmxTransient->fRemoveTscAuxMsr);
11363	pVmxTransient->fRemoveTscAuxMsr = true;
11364	}
11365
11366	#ifdef VBOX_STRICT
11367	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11368	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11369	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11370	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
11371	#endif
11372
11373	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11374	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11375	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11376	* see @bugref{9180#c54}. */
11377	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11378	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11379	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11380	#endif
11381	}
11382
11383
11384	/**
11385	* First C routine invoked after running guest code using hardware-assisted VMX.
11386	*
11387	* @param pVCpu The cross context virtual CPU structure.
11388	* @param pVmxTransient The VMX-transient structure.
11389	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11390	*
11391	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
11392	*
11393	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
11394	* unconditionally when it is safe to do so.
11395	*/
11396	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11397	{
11398	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11399
11400	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11401	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11402	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11403	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11404	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11405	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11406
11407	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11408	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11409	{
11410	uint64_t uGstTsc;
11411	if (!pVmxTransient->fIsNestedGuest)
11412	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11413	else
11414	{
11415	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11416	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11417	}
11418	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11419	}
11420
11421	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11422	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11423	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11424
11425	#if HC_ARCH_BITS == 64
11426	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11427	#endif
11428	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11429	/* The 64-on-32 switcher maintains VMCS-launch state on its own
11430	and we need to leave it alone here. */
11431	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11432	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11433	#else
11434	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11435	#endif
11436	#ifdef VBOX_STRICT
11437	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11438	#endif
11439	Assert(!ASMIntAreEnabled());
11440	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11441	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11442
11443	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
11444	/*
11445	* Clean all the VMCS fields in the transient structure before reading
11446	* anything from the VMCS.
11447	*/
11448	pVmxTransient->uExitReason = 0;
11449	pVmxTransient->uExitIntErrorCode = 0;
11450	pVmxTransient->uExitQual = 0;
11451	pVmxTransient->uGuestLinearAddr = 0;
11452	pVmxTransient->uExitIntInfo = 0;
11453	pVmxTransient->cbInstr = 0;
11454	pVmxTransient->ExitInstrInfo.u = 0;
11455	pVmxTransient->uEntryIntInfo = 0;
11456	pVmxTransient->uEntryXcptErrorCode = 0;
11457	pVmxTransient->cbEntryInstr = 0;
11458	pVmxTransient->uIdtVectoringInfo = 0;
11459	pVmxTransient->uIdtVectoringErrorCode = 0;
11460	#endif
11461
11462	/*
11463	* Save the basic VM-exit reason and check if the VM-entry failed.
11464	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11465	*/
11466	uint32_t uExitReason;
11467	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11468	AssertRC(rc);
11469	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11470	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11471
11472	/*
11473	* Log the VM-exit before logging anything else as otherwise it might be a
11474	* tad confusing what happens before and after the world-switch.
11475	*/
11476	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11477
11478	/*
11479	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11480	* bitmap permissions, if it was added before VM-entry.
11481	*/
11482	if (pVmxTransient->fRemoveTscAuxMsr)
11483	{
11484	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11485	pVmxTransient->fRemoveTscAuxMsr = false;
11486	}
11487
11488	/*
11489	* Check if VMLAUNCH/VMRESUME succeeded.
11490	* If this failed, we cause a guru meditation and cease further execution.
11491	*
11492	* However, if we are executing a nested-guest we might fail if we use the
11493	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11494	*/
11495	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11496	{
11497	/*
11498	* Update the VM-exit history array here even if the VM-entry failed due to:
11499	* - Invalid guest state.
11500	* - MSR loading.
11501	* - Machine-check event.
11502	*
11503	* In any of the above cases we will still have a "valid" VM-exit reason
11504	* despite @a fVMEntryFailed being false.
11505	*
11506	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11507	*
11508	* Note! We don't have CS or RIP at this point. Will probably address that later
11509	* by amending the history entry added here.
11510	*/
11511	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11512	UINT64_MAX, uHostTsc);
11513
11514	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11515	{
11516	VMMRZCallRing3Enable(pVCpu);
11517
11518	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11519	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11520
11521	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11522	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11523	AssertRC(rc);
11524	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11525	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11526	AssertRC(rc);
11527	#else
11528	/*
11529	* Import the guest-interruptibility state always as we need it while evaluating
11530	* injecting events on re-entry.
11531	*
11532	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11533	* checking for real-mode while exporting the state because all bits that cause
11534	* mode changes wrt CR0 are intercepted.
11535	*/
11536	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11537	AssertRC(rc);
11538	#endif
11539
11540	/*
11541	* Sync the TPR shadow with our APIC state.
11542	*
11543	* With nested-guests, mark the virtual-APIC page as dirty so it can be synced
11544	* when performing the nested-guest VM-exit.
11545	*/
11546	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11547	{
11548	if (!pVmxTransient->fIsNestedGuest)
11549	{
11550	Assert(pVmcsInfo->pbVirtApic);
11551	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11552	{
11553	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11554	AssertRC(rc);
11555	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11556	}
11557	}
11558	else
11559	pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtApicPageDirty = true;
11560	}
11561
11562	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11563	return;
11564	}
11565	}
11566	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11567	else if (pVmxTransient->fIsNestedGuest)
11568	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11569	#endif
11570	else
11571	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11572
11573	VMMRZCallRing3Enable(pVCpu);
11574	}
11575
11576
11577	/**
11578	* Runs the guest code using hardware-assisted VMX the normal way.
11579	*
11580	* @returns VBox status code.
11581	* @param pVCpu The cross context virtual CPU structure.
11582	* @param pcLoops Pointer to the number of executed loops.
11583	*/
11584	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11585	{
11586	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11587	Assert(pcLoops);
11588	Assert(*pcLoops <= cMaxResumeLoops);
11589	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11590
11591	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11592	/*
11593	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11594	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11595	* guest VMCS while entering the VMX ring-0 session.
11596	*/
11597	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11598	{
11599	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11600	if (RT_SUCCESS(rc))
11601	{ /* likely */ }
11602	else
11603	{
11604	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11605	return rc;
11606	}
11607	}
11608	#endif
11609
11610	VMXTRANSIENT VmxTransient;
11611	RT_ZERO(VmxTransient);
11612	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11613
11614	/* Paranoia. */
11615	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11616
11617	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11618	for (;;)
11619	{
11620	Assert(!HMR0SuspendPending());
11621	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11622	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11623
11624	/*
11625	* Preparatory work for running nested-guest code, this may force us to
11626	* return to ring-3.
11627	*
11628	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11629	*/
11630	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11631	if (rcStrict != VINF_SUCCESS)
11632	break;
11633
11634	/* Interrupts are disabled at this point! */
11635	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11636	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11637	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11638	/* Interrupts are re-enabled at this point! */
11639
11640	/*
11641	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11642	*/
11643	if (RT_SUCCESS(rcRun))
11644	{ /* very likely */ }
11645	else
11646	{
11647	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11648	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11649	return rcRun;
11650	}
11651
11652	/*
11653	* Profile the VM-exit.
11654	*/
11655	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11656	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11657	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11658	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11659	HMVMX_START_EXIT_DISPATCH_PROF();
11660
11661	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11662
11663	/*
11664	* Handle the VM-exit.
11665	*/
11666	#ifdef HMVMX_USE_FUNCTION_TABLE
11667	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11668	#else
11669	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11670	#endif
11671	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11672	if (rcStrict == VINF_SUCCESS)
11673	{
11674	if (++(*pcLoops) <= cMaxResumeLoops)
11675	continue;
11676	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11677	rcStrict = VINF_EM_RAW_INTERRUPT;
11678	}
11679	break;
11680	}
11681
11682	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11683	return rcStrict;
11684	}
11685
11686
11687	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11688	/**
11689	* Runs the nested-guest code using hardware-assisted VMX.
11690	*
11691	* @returns VBox status code.
11692	* @param pVCpu The cross context virtual CPU structure.
11693	* @param pcLoops Pointer to the number of executed loops.
11694	*
11695	* @sa hmR0VmxRunGuestCodeNormal.
11696	*/
11697	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11698	{
11699	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11700	Assert(pcLoops);
11701	Assert(*pcLoops <= cMaxResumeLoops);
11702	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11703
11704	/*
11705	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11706	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11707	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11708	*/
11709	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11710	{
11711	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11712	if (RT_SUCCESS(rc))
11713	{ /* likely */ }
11714	else
11715	{
11716	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11717	return rc;
11718	}
11719	}
11720
11721	VMXTRANSIENT VmxTransient;
11722	RT_ZERO(VmxTransient);
11723	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11724	VmxTransient.fIsNestedGuest = true;
11725
11726	/* Paranoia. */
11727	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11728
11729	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11730	for (;;)
11731	{
11732	Assert(!HMR0SuspendPending());
11733	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11734	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11735
11736	/*
11737	* Preparatory work for running guest code, this may force us to
11738	* return to ring-3.
11739	*
11740	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11741	*/
11742	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11743	if (rcStrict != VINF_SUCCESS)
11744	break;
11745
11746	/* Interrupts are disabled at this point! */
11747	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11748	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11749	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11750	/* Interrupts are re-enabled at this point! */
11751
11752	/*
11753	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11754	*/
11755	if (RT_SUCCESS(rcRun))
11756	{ /* very likely */ }
11757	else
11758	{
11759	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11760	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11761	return rcRun;
11762	}
11763
11764	/*
11765	* Profile the VM-exit.
11766	*/
11767	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11769	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11770	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11771	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11772	HMVMX_START_EXIT_DISPATCH_PROF();
11773
11774	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11775
11776	/*
11777	* Handle the VM-exit.
11778	*/
11779	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11780	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11781	if (rcStrict == VINF_SUCCESS)
11782	{
11783	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11784	{
11785	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11786	rcStrict = VINF_VMX_VMEXIT;
11787	}
11788	else
11789	{
11790	if (++(*pcLoops) <= cMaxResumeLoops)
11791	continue;
11792	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11793	rcStrict = VINF_EM_RAW_INTERRUPT;
11794	}
11795	}
11796	else
11797	Assert(rcStrict != VINF_VMX_VMEXIT);
11798	break;
11799	}
11800
11801	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11802	return rcStrict;
11803	}
11804	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11805
11806
11807	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11808	* probes.
11809	*
11810	* The following few functions and associated structure contains the bloat
11811	* necessary for providing detailed debug events and dtrace probes as well as
11812	* reliable host side single stepping. This works on the principle of
11813	* "subclassing" the normal execution loop and workers. We replace the loop
11814	* method completely and override selected helpers to add necessary adjustments
11815	* to their core operation.
11816	*
11817	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11818	* any performance for debug and analysis features.
11819	*
11820	* @{
11821	*/
11822
11823	/**
11824	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11825	* the debug run loop.
11826	*/
11827	typedef struct VMXRUNDBGSTATE
11828	{
11829	/** The RIP we started executing at. This is for detecting that we stepped. */
11830	uint64_t uRipStart;
11831	/** The CS we started executing with. */
11832	uint16_t uCsStart;
11833
11834	/** Whether we've actually modified the 1st execution control field. */
11835	bool fModifiedProcCtls : 1;
11836	/** Whether we've actually modified the 2nd execution control field. */
11837	bool fModifiedProcCtls2 : 1;
11838	/** Whether we've actually modified the exception bitmap. */
11839	bool fModifiedXcptBitmap : 1;
11840
11841	/** We desire the modified the CR0 mask to be cleared. */
11842	bool fClearCr0Mask : 1;
11843	/** We desire the modified the CR4 mask to be cleared. */
11844	bool fClearCr4Mask : 1;
11845	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11846	uint32_t fCpe1Extra;
11847	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11848	uint32_t fCpe1Unwanted;
11849	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11850	uint32_t fCpe2Extra;
11851	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11852	uint32_t bmXcptExtra;
11853	/** The sequence number of the Dtrace provider settings the state was
11854	* configured against. */
11855	uint32_t uDtraceSettingsSeqNo;
11856	/** VM-exits to check (one bit per VM-exit). */
11857	uint32_t bmExitsToCheck[3];
11858
11859	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11860	uint32_t fProcCtlsInitial;
11861	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11862	uint32_t fProcCtls2Initial;
11863	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11864	uint32_t bmXcptInitial;
11865	} VMXRUNDBGSTATE;
11866	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11867	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11868
11869
11870	/**
11871	* Initializes the VMXRUNDBGSTATE structure.
11872	*
11873	* @param pVCpu The cross context virtual CPU structure of the
11874	* calling EMT.
11875	* @param pVmxTransient The VMX-transient structure.
11876	* @param pDbgState The debug state to initialize.
11877	*/
11878	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11879	{
11880	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11881	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11882
11883	pDbgState->fModifiedProcCtls = false;
11884	pDbgState->fModifiedProcCtls2 = false;
11885	pDbgState->fModifiedXcptBitmap = false;
11886	pDbgState->fClearCr0Mask = false;
11887	pDbgState->fClearCr4Mask = false;
11888	pDbgState->fCpe1Extra = 0;
11889	pDbgState->fCpe1Unwanted = 0;
11890	pDbgState->fCpe2Extra = 0;
11891	pDbgState->bmXcptExtra = 0;
11892	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11893	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11894	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11895	}
11896
11897
11898	/**
11899	* Updates the VMSC fields with changes requested by @a pDbgState.
11900	*
11901	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11902	* immediately before executing guest code, i.e. when interrupts are disabled.
11903	* We don't check status codes here as we cannot easily assert or return in the
11904	* latter case.
11905	*
11906	* @param pVCpu The cross context virtual CPU structure.
11907	* @param pVmxTransient The VMX-transient structure.
11908	* @param pDbgState The debug state.
11909	*/
11910	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11911	{
11912	/*
11913	* Ensure desired flags in VMCS control fields are set.
11914	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11915	*
11916	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11917	* there should be no stale data in pCtx at this point.
11918	*/
11919	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11920	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11921	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11922	{
11923	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11924	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11925	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11926	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11927	pDbgState->fModifiedProcCtls = true;
11928	}
11929
11930	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11931	{
11932	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11933	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11934	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11935	pDbgState->fModifiedProcCtls2 = true;
11936	}
11937
11938	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11939	{
11940	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11941	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11942	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11943	pDbgState->fModifiedXcptBitmap = true;
11944	}
11945
11946	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11947	{
11948	pVmcsInfo->u64Cr0Mask = 0;
11949	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11950	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11951	}
11952
11953	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11954	{
11955	pVmcsInfo->u64Cr4Mask = 0;
11956	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11957	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11958	}
11959
11960	NOREF(pVCpu);
11961	}
11962
11963
11964	/**
11965	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11966	* re-entry next time around.
11967	*
11968	* @returns Strict VBox status code (i.e. informational status codes too).
11969	* @param pVCpu The cross context virtual CPU structure.
11970	* @param pVmxTransient The VMX-transient structure.
11971	* @param pDbgState The debug state.
11972	* @param rcStrict The return code from executing the guest using single
11973	* stepping.
11974	*/
11975	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11976	VBOXSTRICTRC rcStrict)
11977	{
11978	/*
11979	* Restore VM-exit control settings as we may not reenter this function the
11980	* next time around.
11981	*/
11982	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11983
11984	/* We reload the initial value, trigger what we can of recalculations the
11985	next time around. From the looks of things, that's all that's required atm. */
11986	if (pDbgState->fModifiedProcCtls)
11987	{
11988	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11989	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11990	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11991	AssertRCReturn(rc2, rc2);
11992	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11993	}
11994
11995	/* We're currently the only ones messing with this one, so just restore the
11996	cached value and reload the field. */
11997	if ( pDbgState->fModifiedProcCtls2
11998	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11999	{
12000	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
12001	AssertRCReturn(rc2, rc2);
12002	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
12003	}
12004
12005	/* If we've modified the exception bitmap, we restore it and trigger
12006	reloading and partial recalculation the next time around. */
12007	if (pDbgState->fModifiedXcptBitmap)
12008	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
12009
12010	return rcStrict;
12011	}
12012
12013
12014	/**
12015	* Configures VM-exit controls for current DBGF and DTrace settings.
12016	*
12017	* This updates @a pDbgState and the VMCS execution control fields to reflect
12018	* the necessary VM-exits demanded by DBGF and DTrace.
12019	*
12020	* @param pVCpu The cross context virtual CPU structure.
12021	* @param pVmxTransient The VMX-transient structure. May update
12022	* fUpdatedTscOffsettingAndPreemptTimer.
12023	* @param pDbgState The debug state.
12024	*/
12025	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12026	{
12027	/*
12028	* Take down the dtrace serial number so we can spot changes.
12029	*/
12030	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
12031	ASMCompilerBarrier();
12032
12033	/*
12034	* We'll rebuild most of the middle block of data members (holding the
12035	* current settings) as we go along here, so start by clearing it all.
12036	*/
12037	pDbgState->bmXcptExtra = 0;
12038	pDbgState->fCpe1Extra = 0;
12039	pDbgState->fCpe1Unwanted = 0;
12040	pDbgState->fCpe2Extra = 0;
12041	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
12042	pDbgState->bmExitsToCheck[i] = 0;
12043
12044	/*
12045	* Software interrupts (INT XXh) - no idea how to trigger these...
12046	*/
12047	PVM pVM = pVCpu->CTX_SUFF(pVM);
12048	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
12049	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
12050	{
12051	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12052	}
12053
12054	/*
12055	* INT3 breakpoints - triggered by #BP exceptions.
12056	*/
12057	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
12058	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12059
12060	/*
12061	* Exception bitmap and XCPT events+probes.
12062	*/
12063	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
12064	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
12065	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
12066
12067	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
12068	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
12069	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12070	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
12071	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
12072	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
12073	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
12074	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
12075	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
12076	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
12077	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
12078	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
12079	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
12080	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
12081	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
12082	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
12083	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
12084	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
12085
12086	if (pDbgState->bmXcptExtra)
12087	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12088
12089	/*
12090	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12091	*
12092	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12093	* So, when adding/changing/removing please don't forget to update it.
12094	*
12095	* Some of the macros are picking up local variables to save horizontal space,
12096	* (being able to see it in a table is the lesser evil here).
12097	*/
12098	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12099	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12100	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12101	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12102	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12103	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12104	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12105	} else do { } while (0)
12106	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12107	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12108	{ \
12109	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
12110	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12111	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12112	} else do { } while (0)
12113	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12114	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12115	{ \
12116	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
12117	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12118	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12119	} else do { } while (0)
12120	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12121	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12122	{ \
12123	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
12124	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12125	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12126	} else do { } while (0)
12127
12128	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12129	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12130	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12131	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12132	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12133
12134	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12135	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12136	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12137	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12138	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12139	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12140	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12141	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12142	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12143	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12144	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12145	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12146	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12147	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12148	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12149	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12150	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12151	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12152	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12153	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12154	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12155	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12156	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12157	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12158	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12159	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12160	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12161	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12162	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12163	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12164	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12165	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12166	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12167	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12168	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12169	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12170
12171	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12172	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12173	{
12174	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
12175	\| CPUMCTX_EXTRN_APIC_TPR);
12176	AssertRC(rc);
12177
12178	#if 0 /** @todo fix me */
12179	pDbgState->fClearCr0Mask = true;
12180	pDbgState->fClearCr4Mask = true;
12181	#endif
12182	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12183	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
12184	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12185	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
12186	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12187	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12188	require clearing here and in the loop if we start using it. */
12189	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12190	}
12191	else
12192	{
12193	if (pDbgState->fClearCr0Mask)
12194	{
12195	pDbgState->fClearCr0Mask = false;
12196	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12197	}
12198	if (pDbgState->fClearCr4Mask)
12199	{
12200	pDbgState->fClearCr4Mask = false;
12201	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12202	}
12203	}
12204	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12205	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12206
12207	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12208	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12209	{
12210	/** @todo later, need to fix handler as it assumes this won't usually happen. */
12211	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12212	}
12213	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12214	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12215
12216	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12217	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12218	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12219	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12220	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12221	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12222	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12223	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12224	#if 0 /** @todo too slow, fix handler. */
12225	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12226	#endif
12227	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12228
12229	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12230	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12231	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12232	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12233	{
12234	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12235	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12236	}
12237	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12238	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12240	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12241
12242	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12243	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
12244	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12245	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
12246	{
12247	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12248	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12249	}
12250	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12251	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12252	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12253	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12254
12255	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12256	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12257	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12258	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12259	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12260	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12261	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12262	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12263	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12264	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12265	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12266	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12267	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12268	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12269	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12270	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12271	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12272	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12273	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12274	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12275	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12276	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12277
12278	#undef IS_EITHER_ENABLED
12279	#undef SET_ONLY_XBM_IF_EITHER_EN
12280	#undef SET_CPE1_XBM_IF_EITHER_EN
12281	#undef SET_CPEU_XBM_IF_EITHER_EN
12282	#undef SET_CPE2_XBM_IF_EITHER_EN
12283
12284	/*
12285	* Sanitize the control stuff.
12286	*/
12287	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12288	if (pDbgState->fCpe2Extra)
12289	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12290	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12291	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12292	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12293	{
12294	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12295	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12296	}
12297
12298	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12299	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12300	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12301	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12302	}
12303
12304
12305	/**
12306	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
12307	* appropriate.
12308	*
12309	* The caller has checked the VM-exit against the
12310	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12311	* already, so we don't have to do that either.
12312	*
12313	* @returns Strict VBox status code (i.e. informational status codes too).
12314	* @param pVCpu The cross context virtual CPU structure.
12315	* @param pVmxTransient The VMX-transient structure.
12316	* @param uExitReason The VM-exit reason.
12317	*
12318	* @remarks The name of this function is displayed by dtrace, so keep it short
12319	* and to the point. No longer than 33 chars long, please.
12320	*/
12321	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12322	{
12323	/*
12324	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12325	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
12326	*
12327	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12328	* does. Must add/change/remove both places. Same ordering, please.
12329	*
12330	* Added/removed events must also be reflected in the next section
12331	* where we dispatch dtrace events.
12332	*/
12333	bool fDtrace1 = false;
12334	bool fDtrace2 = false;
12335	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12336	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12337	uint32_t uEventArg = 0;
12338	#define SET_EXIT(a_EventSubName) \
12339	do { \
12340	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12341	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12342	} while (0)
12343	#define SET_BOTH(a_EventSubName) \
12344	do { \
12345	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12346	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12347	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12348	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12349	} while (0)
12350	switch (uExitReason)
12351	{
12352	case VMX_EXIT_MTF:
12353	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12354
12355	case VMX_EXIT_XCPT_OR_NMI:
12356	{
12357	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12358	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12359	{
12360	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12361	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12362	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12363	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12364	{
12365	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12366	{
12367	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12368	uEventArg = pVmxTransient->uExitIntErrorCode;
12369	}
12370	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12371	switch (enmEvent1)
12372	{
12373	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12374	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12375	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12376	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12377	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12378	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12379	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12380	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12381	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12382	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12383	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12384	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12385	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12386	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12387	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12388	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12389	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12390	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12391	default: break;
12392	}
12393	}
12394	else
12395	AssertFailed();
12396	break;
12397
12398	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12399	uEventArg = idxVector;
12400	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12401	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12402	break;
12403	}
12404	break;
12405	}
12406
12407	case VMX_EXIT_TRIPLE_FAULT:
12408	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12409	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12410	break;
12411	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12412	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12413	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12414	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12415	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12416
12417	/* Instruction specific VM-exits: */
12418	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12419	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12420	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12421	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12422	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12423	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12424	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12425	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12426	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12427	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12428	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12429	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12430	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12431	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12432	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12433	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12434	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12435	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12436	case VMX_EXIT_MOV_CRX:
12437	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12438	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12439	SET_BOTH(CRX_READ);
12440	else
12441	SET_BOTH(CRX_WRITE);
12442	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12443	break;
12444	case VMX_EXIT_MOV_DRX:
12445	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12446	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12447	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12448	SET_BOTH(DRX_READ);
12449	else
12450	SET_BOTH(DRX_WRITE);
12451	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12452	break;
12453	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12454	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12455	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12456	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12457	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12458	case VMX_EXIT_GDTR_IDTR_ACCESS:
12459	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12460	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12461	{
12462	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12463	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12464	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12465	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12466	}
12467	break;
12468
12469	case VMX_EXIT_LDTR_TR_ACCESS:
12470	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12471	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12472	{
12473	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12474	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12475	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12476	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12477	}
12478	break;
12479
12480	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12481	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12482	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12483	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12484	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12485	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12486	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12487	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12488	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12489	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12490	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12491
12492	/* Events that aren't relevant at this point. */
12493	case VMX_EXIT_EXT_INT:
12494	case VMX_EXIT_INT_WINDOW:
12495	case VMX_EXIT_NMI_WINDOW:
12496	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12497	case VMX_EXIT_PREEMPT_TIMER:
12498	case VMX_EXIT_IO_INSTR:
12499	break;
12500
12501	/* Errors and unexpected events. */
12502	case VMX_EXIT_INIT_SIGNAL:
12503	case VMX_EXIT_SIPI:
12504	case VMX_EXIT_IO_SMI:
12505	case VMX_EXIT_SMI:
12506	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12507	case VMX_EXIT_ERR_MSR_LOAD:
12508	case VMX_EXIT_ERR_MACHINE_CHECK:
12509	case VMX_EXIT_PML_FULL:
12510	case VMX_EXIT_VIRTUALIZED_EOI:
12511	break;
12512
12513	default:
12514	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12515	break;
12516	}
12517	#undef SET_BOTH
12518	#undef SET_EXIT
12519
12520	/*
12521	* Dtrace tracepoints go first. We do them here at once so we don't
12522	* have to copy the guest state saving and stuff a few dozen times.
12523	* Down side is that we've got to repeat the switch, though this time
12524	* we use enmEvent since the probes are a subset of what DBGF does.
12525	*/
12526	if (fDtrace1 \|\| fDtrace2)
12527	{
12528	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12529	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12530	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12531	switch (enmEvent1)
12532	{
12533	/** @todo consider which extra parameters would be helpful for each probe. */
12534	case DBGFEVENT_END: break;
12535	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12536	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12537	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12538	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12539	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12540	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12541	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12542	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12543	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12544	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12545	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12546	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12547	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12548	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12549	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12550	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12551	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12552	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12553	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12554	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12555	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12556	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12557	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12558	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12559	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12560	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12561	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12562	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12563	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12564	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12565	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12566	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12567	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12568	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12569	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12570	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12571	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12572	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12573	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12574	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12575	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12576	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12577	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12578	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12579	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12580	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12581	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12582	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12583	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12584	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12585	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12586	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12587	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12588	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12589	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12590	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12591	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12592	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12593	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12594	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12595	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12596	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12597	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12598	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12599	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12600	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12601	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12602	}
12603	switch (enmEvent2)
12604	{
12605	/** @todo consider which extra parameters would be helpful for each probe. */
12606	case DBGFEVENT_END: break;
12607	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12608	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12609	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12610	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12611	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12612	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12613	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12614	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12615	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12616	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12617	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12618	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12619	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12620	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12621	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12622	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12623	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12624	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12625	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12626	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12627	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12628	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12629	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12630	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12631	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12632	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12633	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12634	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12635	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12636	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12637	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12638	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12639	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12640	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12641	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12642	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12643	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12644	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12645	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12646	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12647	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12648	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12649	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12650	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12651	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12652	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12653	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12654	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12655	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12656	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12657	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12658	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12659	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12660	}
12661	}
12662
12663	/*
12664	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12665	* the DBGF call will do a full check).
12666	*
12667	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12668	* Note! If we have to events, we prioritize the first, i.e. the instruction
12669	* one, in order to avoid event nesting.
12670	*/
12671	PVM pVM = pVCpu->CTX_SUFF(pVM);
12672	if ( enmEvent1 != DBGFEVENT_END
12673	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12674	{
12675	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12676	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12677	if (rcStrict != VINF_SUCCESS)
12678	return rcStrict;
12679	}
12680	else if ( enmEvent2 != DBGFEVENT_END
12681	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12682	{
12683	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12684	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12685	if (rcStrict != VINF_SUCCESS)
12686	return rcStrict;
12687	}
12688
12689	return VINF_SUCCESS;
12690	}
12691
12692
12693	/**
12694	* Single-stepping VM-exit filtering.
12695	*
12696	* This is preprocessing the VM-exits and deciding whether we've gotten far
12697	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12698	* handling is performed.
12699	*
12700	* @returns Strict VBox status code (i.e. informational status codes too).
12701	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12702	* @param pVmxTransient The VMX-transient structure.
12703	* @param pDbgState The debug state.
12704	*/
12705	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12706	{
12707	/*
12708	* Expensive (saves context) generic dtrace VM-exit probe.
12709	*/
12710	uint32_t const uExitReason = pVmxTransient->uExitReason;
12711	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12712	{ /* more likely */ }
12713	else
12714	{
12715	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12716	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12717	AssertRC(rc);
12718	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12719	}
12720
12721	/*
12722	* Check for host NMI, just to get that out of the way.
12723	*/
12724	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12725	{ /* normally likely */ }
12726	else
12727	{
12728	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12729	AssertRCReturn(rc2, rc2);
12730	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12731	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12732	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12733	}
12734
12735	/*
12736	* Check for single stepping event if we're stepping.
12737	*/
12738	if (pVCpu->hm.s.fSingleInstruction)
12739	{
12740	switch (uExitReason)
12741	{
12742	case VMX_EXIT_MTF:
12743	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12744
12745	/* Various events: */
12746	case VMX_EXIT_XCPT_OR_NMI:
12747	case VMX_EXIT_EXT_INT:
12748	case VMX_EXIT_TRIPLE_FAULT:
12749	case VMX_EXIT_INT_WINDOW:
12750	case VMX_EXIT_NMI_WINDOW:
12751	case VMX_EXIT_TASK_SWITCH:
12752	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12753	case VMX_EXIT_APIC_ACCESS:
12754	case VMX_EXIT_EPT_VIOLATION:
12755	case VMX_EXIT_EPT_MISCONFIG:
12756	case VMX_EXIT_PREEMPT_TIMER:
12757
12758	/* Instruction specific VM-exits: */
12759	case VMX_EXIT_CPUID:
12760	case VMX_EXIT_GETSEC:
12761	case VMX_EXIT_HLT:
12762	case VMX_EXIT_INVD:
12763	case VMX_EXIT_INVLPG:
12764	case VMX_EXIT_RDPMC:
12765	case VMX_EXIT_RDTSC:
12766	case VMX_EXIT_RSM:
12767	case VMX_EXIT_VMCALL:
12768	case VMX_EXIT_VMCLEAR:
12769	case VMX_EXIT_VMLAUNCH:
12770	case VMX_EXIT_VMPTRLD:
12771	case VMX_EXIT_VMPTRST:
12772	case VMX_EXIT_VMREAD:
12773	case VMX_EXIT_VMRESUME:
12774	case VMX_EXIT_VMWRITE:
12775	case VMX_EXIT_VMXOFF:
12776	case VMX_EXIT_VMXON:
12777	case VMX_EXIT_MOV_CRX:
12778	case VMX_EXIT_MOV_DRX:
12779	case VMX_EXIT_IO_INSTR:
12780	case VMX_EXIT_RDMSR:
12781	case VMX_EXIT_WRMSR:
12782	case VMX_EXIT_MWAIT:
12783	case VMX_EXIT_MONITOR:
12784	case VMX_EXIT_PAUSE:
12785	case VMX_EXIT_GDTR_IDTR_ACCESS:
12786	case VMX_EXIT_LDTR_TR_ACCESS:
12787	case VMX_EXIT_INVEPT:
12788	case VMX_EXIT_RDTSCP:
12789	case VMX_EXIT_INVVPID:
12790	case VMX_EXIT_WBINVD:
12791	case VMX_EXIT_XSETBV:
12792	case VMX_EXIT_RDRAND:
12793	case VMX_EXIT_INVPCID:
12794	case VMX_EXIT_VMFUNC:
12795	case VMX_EXIT_RDSEED:
12796	case VMX_EXIT_XSAVES:
12797	case VMX_EXIT_XRSTORS:
12798	{
12799	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12800	AssertRCReturn(rc, rc);
12801	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12802	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12803	return VINF_EM_DBG_STEPPED;
12804	break;
12805	}
12806
12807	/* Errors and unexpected events: */
12808	case VMX_EXIT_INIT_SIGNAL:
12809	case VMX_EXIT_SIPI:
12810	case VMX_EXIT_IO_SMI:
12811	case VMX_EXIT_SMI:
12812	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12813	case VMX_EXIT_ERR_MSR_LOAD:
12814	case VMX_EXIT_ERR_MACHINE_CHECK:
12815	case VMX_EXIT_PML_FULL:
12816	case VMX_EXIT_VIRTUALIZED_EOI:
12817	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12818	break;
12819
12820	default:
12821	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12822	break;
12823	}
12824	}
12825
12826	/*
12827	* Check for debugger event breakpoints and dtrace probes.
12828	*/
12829	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12830	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12831	{
12832	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12833	if (rcStrict != VINF_SUCCESS)
12834	return rcStrict;
12835	}
12836
12837	/*
12838	* Normal processing.
12839	*/
12840	#ifdef HMVMX_USE_FUNCTION_TABLE
12841	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12842	#else
12843	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12844	#endif
12845	}
12846
12847
12848	/**
12849	* Single steps guest code using hardware-assisted VMX.
12850	*
12851	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12852	* but single-stepping through the hypervisor debugger.
12853	*
12854	* @returns Strict VBox status code (i.e. informational status codes too).
12855	* @param pVCpu The cross context virtual CPU structure.
12856	* @param pcLoops Pointer to the number of executed loops.
12857	*
12858	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12859	*/
12860	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12861	{
12862	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12863	Assert(pcLoops);
12864	Assert(*pcLoops <= cMaxResumeLoops);
12865
12866	VMXTRANSIENT VmxTransient;
12867	RT_ZERO(VmxTransient);
12868	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12869
12870	/* Set HMCPU indicators. */
12871	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12872	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12873	pVCpu->hm.s.fDebugWantRdTscExit = false;
12874	pVCpu->hm.s.fUsingDebugLoop = true;
12875
12876	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12877	VMXRUNDBGSTATE DbgState;
12878	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12879	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12880
12881	/*
12882	* The loop.
12883	*/
12884	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12885	for (;;)
12886	{
12887	Assert(!HMR0SuspendPending());
12888	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12889	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12890	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12891
12892	/* Set up VM-execution controls the next two can respond to. */
12893	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12894
12895	/*
12896	* Preparatory work for running guest code, this may force us to
12897	* return to ring-3.
12898	*
12899	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12900	*/
12901	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12902	if (rcStrict != VINF_SUCCESS)
12903	break;
12904
12905	/* Interrupts are disabled at this point! */
12906	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12907
12908	/* Override any obnoxious code in the above two calls. */
12909	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12910
12911	/*
12912	* Finally execute the guest.
12913	*/
12914	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12915
12916	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12917	/* Interrupts are re-enabled at this point! */
12918
12919	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12920	if (RT_SUCCESS(rcRun))
12921	{ /* very likely */ }
12922	else
12923	{
12924	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12925	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12926	return rcRun;
12927	}
12928
12929	/* Profile the VM-exit. */
12930	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12932	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12933	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12934	HMVMX_START_EXIT_DISPATCH_PROF();
12935
12936	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12937
12938	/*
12939	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12940	*/
12941	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12942	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12943	if (rcStrict != VINF_SUCCESS)
12944	break;
12945	if (++(*pcLoops) > cMaxResumeLoops)
12946	{
12947	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12948	rcStrict = VINF_EM_RAW_INTERRUPT;
12949	break;
12950	}
12951
12952	/*
12953	* Stepping: Did the RIP change, if so, consider it a single step.
12954	* Otherwise, make sure one of the TFs gets set.
12955	*/
12956	if (fStepping)
12957	{
12958	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12959	AssertRC(rc);
12960	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12961	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12962	{
12963	rcStrict = VINF_EM_DBG_STEPPED;
12964	break;
12965	}
12966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12967	}
12968
12969	/*
12970	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12971	*/
12972	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12973	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12974	}
12975
12976	/*
12977	* Clear the X86_EFL_TF if necessary.
12978	*/
12979	if (pVCpu->hm.s.fClearTrapFlag)
12980	{
12981	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12982	AssertRC(rc);
12983	pVCpu->hm.s.fClearTrapFlag = false;
12984	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12985	}
12986	/** @todo there seems to be issues with the resume flag when the monitor trap
12987	* flag is pending without being used. Seen early in bios init when
12988	* accessing APIC page in protected mode. */
12989
12990	/*
12991	* Restore VM-exit control settings as we may not re-enter this function the
12992	* next time around.
12993	*/
12994	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12995
12996	/* Restore HMCPU indicators. */
12997	pVCpu->hm.s.fUsingDebugLoop = false;
12998	pVCpu->hm.s.fDebugWantRdTscExit = false;
12999	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
13000
13001	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
13002	return rcStrict;
13003	}
13004
13005
13006	/** @} */
13007
13008
13009	/**
13010	* Checks if any expensive dtrace probes are enabled and we should go to the
13011	* debug loop.
13012	*
13013	* @returns true if we should use debug loop, false if not.
13014	*/
13015	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
13016	{
13017	/* It's probably faster to OR the raw 32-bit counter variables together.
13018	Since the variables are in an array and the probes are next to one
13019	another (more or less), we have good locality. So, better read
13020	eight-nine cache lines ever time and only have one conditional, than
13021	128+ conditionals, right? */
13022	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
13023	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
13024	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
13025	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
13026	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
13027	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
13028	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
13029	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
13030	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
13031	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
13032	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
13033	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
13034	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
13035	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
13036	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
13037	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
13038	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
13039	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
13040	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
13041	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
13042	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
13043	) != 0
13044	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
13045	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
13046	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
13047	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
13048	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
13049	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
13050	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
13051	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
13052	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
13053	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
13054	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
13055	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
13056	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
13057	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
13058	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
13059	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
13060	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
13061	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
13062	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
13063	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
13064	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
13065	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
13066	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
13067	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
13068	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
13069	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
13070	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
13071	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
13072	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
13073	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13074	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13075	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13076	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13077	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13078	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13079	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13080	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13081	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13082	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13083	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13084	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13085	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13086	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13087	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13088	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13089	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13090	) != 0
13091	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13092	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
13093	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13094	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13095	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13096	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
13097	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13098	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13099	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13100	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13101	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13102	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13103	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13104	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13105	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13106	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13107	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13108	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13109	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13110	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13111	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13112	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13113	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13114	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13115	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13116	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
13117	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
13118	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13119	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
13120	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13121	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13122	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13123	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13124	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13125	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13126	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13127	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13128	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13129	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13130	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13131	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13132	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13133	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13134	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13135	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13136	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13137	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13138	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13139	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13140	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13141	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13142	) != 0;
13143	}
13144
13145
13146	/**
13147	* Runs the guest using hardware-assisted VMX.
13148	*
13149	* @returns Strict VBox status code (i.e. informational status codes too).
13150	* @param pVCpu The cross context virtual CPU structure.
13151	*/
13152	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13153	{
13154	AssertPtr(pVCpu);
13155	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13156	Assert(VMMRZCallRing3IsEnabled(pVCpu));
13157	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13158	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13159
13160	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13161
13162	VBOXSTRICTRC rcStrict;
13163	uint32_t cLoops = 0;
13164	for (;;)
13165	{
13166	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13167	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13168	#else
13169	bool const fInNestedGuestMode = false;
13170	#endif
13171	if (!fInNestedGuestMode)
13172	{
13173	if ( !pVCpu->hm.s.fUseDebugLoop
13174	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
13175	&& !DBGFIsStepping(pVCpu)
13176	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13177	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13178	else
13179	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13180	}
13181	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13182	else
13183	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13184
13185	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13186	{
13187	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
13188	continue;
13189	}
13190	if (rcStrict == VINF_VMX_VMEXIT)
13191	{
13192	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
13193	continue;
13194	}
13195	#endif
13196	break;
13197	}
13198
13199	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13200	switch (rcLoop)
13201	{
13202	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13203	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13204	}
13205
13206	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13207	if (RT_FAILURE(rc2))
13208	{
13209	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13210	rcStrict = rc2;
13211	}
13212	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13213	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13214	return rcStrict;
13215	}
13216
13217
13218	#ifndef HMVMX_USE_FUNCTION_TABLE
13219	/**
13220	* Handles a guest VM-exit from hardware-assisted VMX execution.
13221	*
13222	* @returns Strict VBox status code (i.e. informational status codes too).
13223	* @param pVCpu The cross context virtual CPU structure.
13224	* @param pVmxTransient The VMX-transient structure.
13225	*/
13226	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13227	{
13228	#ifdef DEBUG_ramshankar
13229	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13230	do { \
13231	if (a_fSave != 0) \
13232	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
13233	VBOXSTRICTRC rcStrict = a_CallExpr; \
13234	if (a_fSave != 0) \
13235	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13236	return rcStrict; \
13237	} while (0)
13238	#else
13239	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13240	#endif
13241	uint32_t const uExitReason = pVmxTransient->uExitReason;
13242	switch (uExitReason)
13243	{
13244	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13245	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13246	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13247	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13248	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13249	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13250	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13251	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13252	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13253	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13254	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13255	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13256	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13257	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13258	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13259	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13260	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13261	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13262	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13263	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13264	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13265	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13266	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13267	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13268	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13269	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13270	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13271	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13272	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13273	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13274	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13275	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13276	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13277	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13278	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13279	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13280	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13281	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13282	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13283	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13284	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13285	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13286	#else
13287	case VMX_EXIT_VMCLEAR:
13288	case VMX_EXIT_VMLAUNCH:
13289	case VMX_EXIT_VMPTRLD:
13290	case VMX_EXIT_VMPTRST:
13291	case VMX_EXIT_VMREAD:
13292	case VMX_EXIT_VMRESUME:
13293	case VMX_EXIT_VMWRITE:
13294	case VMX_EXIT_VMXOFF:
13295	case VMX_EXIT_VMXON:
13296	case VMX_EXIT_INVVPID:
13297	case VMX_EXIT_INVEPT:
13298	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13299	#endif
13300
13301	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13302	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13303	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13304
13305	case VMX_EXIT_INIT_SIGNAL:
13306	case VMX_EXIT_SIPI:
13307	case VMX_EXIT_IO_SMI:
13308	case VMX_EXIT_SMI:
13309	case VMX_EXIT_ERR_MSR_LOAD:
13310	case VMX_EXIT_ERR_MACHINE_CHECK:
13311	case VMX_EXIT_PML_FULL:
13312	case VMX_EXIT_VIRTUALIZED_EOI:
13313	case VMX_EXIT_GDTR_IDTR_ACCESS:
13314	case VMX_EXIT_LDTR_TR_ACCESS:
13315	case VMX_EXIT_APIC_WRITE:
13316	case VMX_EXIT_RDRAND:
13317	case VMX_EXIT_RSM:
13318	case VMX_EXIT_VMFUNC:
13319	case VMX_EXIT_ENCLS:
13320	case VMX_EXIT_RDSEED:
13321	case VMX_EXIT_XSAVES:
13322	case VMX_EXIT_XRSTORS:
13323	case VMX_EXIT_UMWAIT:
13324	case VMX_EXIT_TPAUSE:
13325	default:
13326	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13327	}
13328	#undef VMEXIT_CALL_RET
13329	}
13330	#endif /* !HMVMX_USE_FUNCTION_TABLE */
13331
13332
13333	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13334	/**
13335	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13336	*
13337	* @returns Strict VBox status code (i.e. informational status codes too).
13338	* @param pVCpu The cross context virtual CPU structure.
13339	* @param pVmxTransient The VMX-transient structure.
13340	*/
13341	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13342	{
13343	uint32_t const uExitReason = pVmxTransient->uExitReason;
13344	switch (uExitReason)
13345	{
13346	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13347	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13348	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13349	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13350	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13351
13352	/*
13353	* We shouldn't direct host physical interrupts to the nested-guest.
13354	*/
13355	case VMX_EXIT_EXT_INT:
13356	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13357
13358	/*
13359	* Instructions that cause VM-exits unconditionally or the condition is
13360	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13361	* happens, it's guaranteed to be a nested-guest VM-exit).
13362	*
13363	* - Provides VM-exit instruction length ONLY.
13364	*/
13365	case VMX_EXIT_CPUID: /* Unconditional. */
13366	case VMX_EXIT_VMCALL:
13367	case VMX_EXIT_GETSEC:
13368	case VMX_EXIT_INVD:
13369	case VMX_EXIT_XSETBV:
13370	case VMX_EXIT_VMLAUNCH:
13371	case VMX_EXIT_VMRESUME:
13372	case VMX_EXIT_VMXOFF:
13373	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13374	case VMX_EXIT_VMFUNC:
13375	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13376
13377	/*
13378	* Instructions that cause VM-exits unconditionally or the condition is
13379	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13380	* happens, it's guaranteed to be a nested-guest VM-exit).
13381	*
13382	* - Provides VM-exit instruction length.
13383	* - Provides VM-exit information.
13384	* - Optionally provides Exit qualification.
13385	*
13386	* Since Exit qualification is 0 for all VM-exits where it is not
13387	* applicable, reading and passing it to the guest should produce
13388	* defined behavior.
13389	*
13390	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
13391	*/
13392	case VMX_EXIT_INVEPT: /* Unconditional. */
13393	case VMX_EXIT_INVVPID:
13394	case VMX_EXIT_VMCLEAR:
13395	case VMX_EXIT_VMPTRLD:
13396	case VMX_EXIT_VMPTRST:
13397	case VMX_EXIT_VMXON:
13398	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13399	case VMX_EXIT_LDTR_TR_ACCESS:
13400	case VMX_EXIT_RDRAND:
13401	case VMX_EXIT_RDSEED:
13402	case VMX_EXIT_XSAVES:
13403	case VMX_EXIT_XRSTORS:
13404	case VMX_EXIT_UMWAIT:
13405	case VMX_EXIT_TPAUSE:
13406	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13407
13408	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13409	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13410	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13411	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13412	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13413	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13414	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13415	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13416	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13417	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13418	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13419	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13420	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13421	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13422	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13423	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13424	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13425	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13426	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13427
13428	case VMX_EXIT_PREEMPT_TIMER:
13429	{
13430	/** @todo NSTVMX: Preempt timer. */
13431	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
13432	}
13433
13434	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13435	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13436
13437	case VMX_EXIT_VMREAD:
13438	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13439
13440	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13441	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13442
13443	case VMX_EXIT_INIT_SIGNAL:
13444	case VMX_EXIT_SIPI:
13445	case VMX_EXIT_IO_SMI:
13446	case VMX_EXIT_SMI:
13447	case VMX_EXIT_ERR_MSR_LOAD:
13448	case VMX_EXIT_ERR_MACHINE_CHECK:
13449	case VMX_EXIT_PML_FULL:
13450	case VMX_EXIT_RSM:
13451	default:
13452	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13453	}
13454	}
13455	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13456
13457
13458	/** @name VM-exit helpers.
13459	* @{
13460	*/
13461	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13462	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13463	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13464
13465	/** Macro for VM-exits called unexpectedly. */
13466	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13467	do { \
13468	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13469	return VERR_VMX_UNEXPECTED_EXIT; \
13470	} while (0)
13471
13472	#ifdef VBOX_STRICT
13473	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13474	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13475	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13476
13477	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13478	do { \
13479	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13480	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13481	} while (0)
13482
13483	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13484	do { \
13485	AssertPtr((a_pVCpu)); \
13486	AssertPtr((a_pVmxTransient)); \
13487	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13488	Assert((a_pVmxTransient)->pVmcsInfo); \
13489	Assert(ASMIntAreEnabled()); \
13490	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13491	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13492	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13493	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13494	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13495	HMVMX_ASSERT_PREEMPT_CPUID(); \
13496	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13497	} while (0)
13498
13499	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13500	do { \
13501	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13502	Assert((a_pVmxTransient)->fIsNestedGuest); \
13503	} while (0)
13504
13505	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13506	do { \
13507	Log4Func(("\n")); \
13508	} while (0)
13509	#else
13510	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13511	do { \
13512	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13513	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13514	} while (0)
13515
13516	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13517	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13518
13519	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13520	#endif
13521
13522	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13523	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13524	* guests that attempted to execute a VMX instruction. */
13525	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13526	do \
13527	{ \
13528	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13529	if (rcStrictTmp == VINF_SUCCESS) \
13530	{ /* likely */ } \
13531	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13532	{ \
13533	Assert((a_pVCpu)->hm.s.Event.fPending); \
13534	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13535	return VINF_SUCCESS; \
13536	} \
13537	else \
13538	{ \
13539	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13540	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13541	} \
13542	} while (0)
13543
13544	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13545	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13546	do \
13547	{ \
13548	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13549	(a_pGCPtrEffAddr)); \
13550	if (rcStrictTmp == VINF_SUCCESS) \
13551	{ /* likely */ } \
13552	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13553	{ \
13554	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13555	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13556	NOREF(uXcptTmp); \
13557	return VINF_SUCCESS; \
13558	} \
13559	else \
13560	{ \
13561	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13562	return rcStrictTmp; \
13563	} \
13564	} while (0)
13565	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13566
13567
13568	/**
13569	* Advances the guest RIP by the specified number of bytes.
13570	*
13571	* @param pVCpu The cross context virtual CPU structure.
13572	* @param cbInstr Number of bytes to advance the RIP by.
13573	*
13574	* @remarks No-long-jump zone!!!
13575	*/
13576	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13577	{
13578	/* Advance the RIP. */
13579	pVCpu->cpum.GstCtx.rip += cbInstr;
13580	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13581
13582	/* Update interrupt inhibition. */
13583	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13584	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13585	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13586	}
13587
13588
13589	/**
13590	* Advances the guest RIP after reading it from the VMCS.
13591	*
13592	* @returns VBox status code, no informational status codes.
13593	* @param pVCpu The cross context virtual CPU structure.
13594	* @param pVmxTransient The VMX-transient structure.
13595	*
13596	* @remarks No-long-jump zone!!!
13597	*/
13598	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13599	{
13600	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13601	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13602	AssertRCReturn(rc, rc);
13603
13604	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13605	return VINF_SUCCESS;
13606	}
13607
13608
13609	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13610	/**
13611	* Handle a condition that occurred while delivering an event through the
13612	* nested-guest IDT.
13613	*
13614	* @returns VBox status code.
13615	* @param pVCpu The cross context virtual CPU structure.
13616	* @param pVmxTransient The VMX-transient structure.
13617	*
13618	* @remarks No-long-jump zone!!!
13619	*/
13620	static int hmR0VmxCheckExitDueToEventDeliveryNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13621	{
13622	Assert(pVmxTransient->fIsNestedGuest);
13623	Assert(!pVCpu->hm.s.Event.fPending);
13624
13625	/*
13626	* Construct a pending event from IDT vectoring information.
13627	*
13628	* This event could have originated from an event that we or the guest hypervisor injected
13629	* during nested-guest VM-entry or could arise from hardware-assisted VMX execution of the
13630	* nested-guest (for e.g. a #GP fault causing a #PF VM-exit).
13631	*
13632	* If the VM-exit is caused indirectly due to delivery of:
13633	* - #PF: the CPU would have updated CR2.
13634	* - NMI: NMI/virtual-NMI blocking is in effect.
13635	*
13636	* The main differences between this function and its non-nested version are as follows:
13637	*
13638	* - Here we record software interrupts, software exceptions and privileged software
13639	* exceptions as pending for re-injection when necessary along with gathering the
13640	* instruction length. The non-nested version would fix-up the VM-exit that occurred
13641	* during delivery of such an event and restart execution of the guest without
13642	* re-injecting the event and does not record the instruction length.
13643	*
13644	* - Here we record #PF as pending for re-injection while the non-nested version would
13645	* handle it via the page-fault VM-exit handler which isn't required when nested paging
13646	* is a requirement for hardware-assisted VMX execution of nested-guests.
13647	*
13648	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
13649	*/
13650	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13651	AssertRCReturn(rc, rc);
13652
13653	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13654	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13655	{
13656	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13657	uint8_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13658
13659	/*
13660	* Get the nasty stuff out of the way.
13661	*/
13662	{
13663	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13664	AssertRCReturn(rc, rc);
13665
13666	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13667	if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13668	{
13669	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13670	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13671	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13672
13673	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13674	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13675
13676	IEMXCPTRAISEINFO fRaiseInfo;
13677	IEMXCPTRAISE const enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags,
13678	uExitVector, &fRaiseInfo);
13679	if (enmRaise == IEMXCPTRAISE_CPU_HANG)
13680	{
13681	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13682	return VERR_EM_GUEST_CPU_HANG;
13683	}
13684	}
13685	}
13686
13687	/*
13688	* Things look legit, continue...
13689	*/
13690	uint32_t u32ErrCode;
13691	bool const fErrCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo);
13692	if (fErrCodeValid)
13693	{
13694	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13695	AssertRCReturn(rc, rc);
13696	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13697	}
13698	else
13699	u32ErrCode = 0;
13700
13701	uint32_t cbInstr;
13702	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13703	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT
13704	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT)
13705	{
13706	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13707	AssertRCReturn(rc, rc);
13708	cbInstr = pVmxTransient->cbInstr;
13709	}
13710	else
13711	cbInstr = 0;
13712
13713	RTGCUINTPTR GCPtrFaultAddress;
13714	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(uIdtVectorInfo))
13715	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
13716	else
13717	GCPtrFaultAddress = 0;
13718
13719	if (VMX_IDT_VECTORING_INFO_IS_XCPT_NMI(uIdtVectorInfo))
13720	CPUMSetGuestNmiBlocking(pVCpu, true);
13721
13722	hmR0VmxSetPendingEvent(pVCpu, uIdtVectorInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
13723	}
13724
13725	return VINF_SUCCESS;
13726	}
13727	#endif
13728
13729
13730	/**
13731	* Handle a condition that occurred while delivering an event through the guest or
13732	* nested-guest IDT.
13733	*
13734	* @returns Strict VBox status code (i.e. informational status codes too).
13735	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13736	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13737	* to continue execution of the guest which will delivery the \#DF.
13738	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13739	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13740	*
13741	* @param pVCpu The cross context virtual CPU structure.
13742	* @param pVmxTransient The VMX-transient structure.
13743	*
13744	* @remarks No-long-jump zone!!!
13745	*/
13746	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13747	{
13748	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13749	if (pVmxTransient->fIsNestedGuest)
13750	return hmR0VmxCheckExitDueToEventDeliveryNested(pVCpu, pVmxTransient);
13751	#endif
13752
13753	Assert(!pVmxTransient->fIsNestedGuest);
13754	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13755
13756	/* Read the IDT vectoring info. and VM-exit interruption info. */
13757	{
13758	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13759	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13760	AssertRCReturn(rc, rc);
13761	}
13762
13763	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13764	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13765	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13766	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13767	{
13768	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13769	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13770
13771	/*
13772	* If the event was a software interrupt (generated with INT n) or a software exception
13773	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13774	* can handle the VM-exit and continue guest execution which will re-execute the
13775	* instruction rather than re-injecting the exception, as that can cause premature
13776	* trips to ring-3 before injection and involve TRPM which currently has no way of
13777	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13778	* the problem).
13779	*/
13780	IEMXCPTRAISE enmRaise;
13781	IEMXCPTRAISEINFO fRaiseInfo;
13782	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13783	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13784	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13785	{
13786	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13787	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13788	}
13789	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13790	{
13791	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13792	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13793
13794	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13795	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13796
13797	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13798
13799	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13800	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13801	{
13802	pVmxTransient->fVectoringPF = true;
13803	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13804	}
13805	}
13806	else
13807	{
13808	/*
13809	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13810	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13811	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13812	*/
13813	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13814	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13815	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13816	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13817	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13818	}
13819
13820	/*
13821	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13822	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13823	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13824	* subsequent VM-entry would fail, see @bugref{7445}.
13825	*
13826	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13827	*/
13828	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13829	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13830	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13831	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13832	&& CPUMIsGuestNmiBlocking(pVCpu))
13833	{
13834	CPUMSetGuestNmiBlocking(pVCpu, false);
13835	}
13836
13837	switch (enmRaise)
13838	{
13839	case IEMXCPTRAISE_CURRENT_XCPT:
13840	{
13841	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", uIdtVectorInfo,
13842	uExitIntInfo));
13843	Assert(rcStrict == VINF_SUCCESS);
13844	break;
13845	}
13846
13847	case IEMXCPTRAISE_PREV_EVENT:
13848	{
13849	uint32_t u32ErrCode;
13850	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13851	{
13852	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13853	AssertRCReturn(rc, rc);
13854	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13855	}
13856	else
13857	u32ErrCode = 0;
13858
13859	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13860	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13861	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13862	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13863
13864	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13865	pVCpu->hm.s.Event.u32ErrCode));
13866	Assert(rcStrict == VINF_SUCCESS);
13867	break;
13868	}
13869
13870	case IEMXCPTRAISE_REEXEC_INSTR:
13871	Assert(rcStrict == VINF_SUCCESS);
13872	break;
13873
13874	case IEMXCPTRAISE_DOUBLE_FAULT:
13875	{
13876	/*
13877	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13878	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13879	*/
13880	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13881	{
13882	pVmxTransient->fVectoringDoublePF = true;
13883	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13884	pVCpu->cpum.GstCtx.cr2));
13885	rcStrict = VINF_SUCCESS;
13886	}
13887	else
13888	{
13889	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13890	hmR0VmxSetPendingXcptDF(pVCpu);
13891	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13892	uIdtVector, uExitVector));
13893	rcStrict = VINF_HM_DOUBLE_FAULT;
13894	}
13895	break;
13896	}
13897
13898	case IEMXCPTRAISE_TRIPLE_FAULT:
13899	{
13900	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13901	rcStrict = VINF_EM_RESET;
13902	break;
13903	}
13904
13905	case IEMXCPTRAISE_CPU_HANG:
13906	{
13907	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13908	rcStrict = VERR_EM_GUEST_CPU_HANG;
13909	break;
13910	}
13911
13912	default:
13913	{
13914	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13915	rcStrict = VERR_VMX_IPE_2;
13916	break;
13917	}
13918	}
13919	}
13920	else if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13921	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo)
13922	&& uExitVector != X86_XCPT_DF
13923	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13924	{
13925	Assert(!VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo));
13926
13927	/*
13928	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13929	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13930	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13931	*/
13932	CPUMSetGuestNmiBlocking(pVCpu, true);
13933	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13934	}
13935
13936	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13937	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13938	return rcStrict;
13939	}
13940
13941
13942	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13943	/**
13944	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13945	* guest attempting to execute a VMX instruction.
13946	*
13947	* @returns Strict VBox status code (i.e. informational status codes too).
13948	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13949	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13950	*
13951	* @param pVCpu The cross context virtual CPU structure.
13952	* @param uExitReason The VM-exit reason.
13953	*
13954	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13955	* @remarks No-long-jump zone!!!
13956	*/
13957	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13958	{
13959	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13960	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13961
13962	/*
13963	* The physical CPU would have already checked the CPU mode/code segment.
13964	* We shall just assert here for paranoia.
13965	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13966	*/
13967	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13968	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13969	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13970
13971	if (uExitReason == VMX_EXIT_VMXON)
13972	{
13973	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13974
13975	/*
13976	* We check CR4.VMXE because it is required to be always set while in VMX operation
13977	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13978	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13979	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13980	*/
13981	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13982	{
13983	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13984	hmR0VmxSetPendingXcptUD(pVCpu);
13985	return VINF_HM_PENDING_XCPT;
13986	}
13987	}
13988	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13989	{
13990	/*
13991	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13992	* (other than VMXON), we need to raise a #UD.
13993	*/
13994	Log4Func(("Not in VMX root mode -> #UD\n"));
13995	hmR0VmxSetPendingXcptUD(pVCpu);
13996	return VINF_HM_PENDING_XCPT;
13997	}
13998
13999	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
14000	return VINF_SUCCESS;
14001	}
14002
14003	/**
14004	* Decodes the memory operand of an instruction that caused a VM-exit.
14005	*
14006	* The Exit qualification field provides the displacement field for memory
14007	* operand instructions, if any.
14008	*
14009	* @returns Strict VBox status code (i.e. informational status codes too).
14010	* @retval VINF_SUCCESS if the operand was successfully decoded.
14011	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
14012	* operand.
14013	* @param pVCpu The cross context virtual CPU structure.
14014	* @param uExitInstrInfo The VM-exit instruction information field.
14015	* @param enmMemAccess The memory operand's access type (read or write).
14016	* @param GCPtrDisp The instruction displacement field, if any. For
14017	* RIP-relative addressing pass RIP + displacement here.
14018	* @param pGCPtrMem Where to store the effective destination memory address.
14019	*
14020	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
14021	* virtual-8086 mode hence skips those checks while verifying if the
14022	* segment is valid.
14023	*/
14024	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
14025	PRTGCPTR pGCPtrMem)
14026	{
14027	Assert(pGCPtrMem);
14028	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
14029	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
14030	\| CPUMCTX_EXTRN_CR0);
14031
14032	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
14033	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
14034	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
14035
14036	VMXEXITINSTRINFO ExitInstrInfo;
14037	ExitInstrInfo.u = uExitInstrInfo;
14038	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
14039	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
14040	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
14041	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
14042	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
14043	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
14044	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
14045	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
14046	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
14047
14048	/*
14049	* Validate instruction information.
14050	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
14051	*/
14052	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
14053	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
14054	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
14055	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
14056	AssertLogRelMsgReturn(fIsMemOperand,
14057	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
14058
14059	/*
14060	* Compute the complete effective address.
14061	*
14062	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
14063	* See AMD spec. 4.5.2 "Segment Registers".
14064	*/
14065	RTGCPTR GCPtrMem = GCPtrDisp;
14066	if (fBaseRegValid)
14067	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
14068	if (fIdxRegValid)
14069	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
14070
14071	RTGCPTR const GCPtrOff = GCPtrMem;
14072	if ( !fIsLongMode
14073	\|\| iSegReg >= X86_SREG_FS)
14074	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
14075	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
14076
14077	/*
14078	* Validate effective address.
14079	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
14080	*/
14081	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
14082	Assert(cbAccess > 0);
14083	if (fIsLongMode)
14084	{
14085	if (X86_IS_CANONICAL(GCPtrMem))
14086	{
14087	*pGCPtrMem = GCPtrMem;
14088	return VINF_SUCCESS;
14089	}
14090
14091	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
14092	* "Data Limit Checks in 64-bit Mode". */
14093	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
14094	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14095	return VINF_HM_PENDING_XCPT;
14096	}
14097
14098	/*
14099	* This is a watered down version of iemMemApplySegment().
14100	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
14101	* and segment CPL/DPL checks are skipped.
14102	*/
14103	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
14104	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
14105	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
14106
14107	/* Check if the segment is present and usable. */
14108	if ( pSel->Attr.n.u1Present
14109	&& !pSel->Attr.n.u1Unusable)
14110	{
14111	Assert(pSel->Attr.n.u1DescType);
14112	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
14113	{
14114	/* Check permissions for the data segment. */
14115	if ( enmMemAccess == VMXMEMACCESS_WRITE
14116	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
14117	{
14118	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14119	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
14120	return VINF_HM_PENDING_XCPT;
14121	}
14122
14123	/* Check limits if it's a normal data segment. */
14124	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
14125	{
14126	if ( GCPtrFirst32 > pSel->u32Limit
14127	\|\| GCPtrLast32 > pSel->u32Limit)
14128	{
14129	Log4Func(("Data segment limit exceeded. "
14130	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14131	GCPtrLast32, pSel->u32Limit));
14132	if (iSegReg == X86_SREG_SS)
14133	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14134	else
14135	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14136	return VINF_HM_PENDING_XCPT;
14137	}
14138	}
14139	else
14140	{
14141	/* Check limits if it's an expand-down data segment.
14142	Note! The upper boundary is defined by the B bit, not the G bit! */
14143	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
14144	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
14145	{
14146	Log4Func(("Expand-down data segment limit exceeded. "
14147	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14148	GCPtrLast32, pSel->u32Limit));
14149	if (iSegReg == X86_SREG_SS)
14150	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14151	else
14152	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14153	return VINF_HM_PENDING_XCPT;
14154	}
14155	}
14156	}
14157	else
14158	{
14159	/* Check permissions for the code segment. */
14160	if ( enmMemAccess == VMXMEMACCESS_WRITE
14161	\|\| ( enmMemAccess == VMXMEMACCESS_READ
14162	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
14163	{
14164	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
14165	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
14166	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14167	return VINF_HM_PENDING_XCPT;
14168	}
14169
14170	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
14171	if ( GCPtrFirst32 > pSel->u32Limit
14172	\|\| GCPtrLast32 > pSel->u32Limit)
14173	{
14174	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
14175	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
14176	if (iSegReg == X86_SREG_SS)
14177	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14178	else
14179	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14180	return VINF_HM_PENDING_XCPT;
14181	}
14182	}
14183	}
14184	else
14185	{
14186	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14187	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14188	return VINF_HM_PENDING_XCPT;
14189	}
14190
14191	*pGCPtrMem = GCPtrMem;
14192	return VINF_SUCCESS;
14193	}
14194	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14195
14196
14197	/**
14198	* VM-exit helper for LMSW.
14199	*/
14200	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
14201	{
14202	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14203	AssertRCReturn(rc, rc);
14204
14205	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14206	AssertMsg( rcStrict == VINF_SUCCESS
14207	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14208
14209	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14210	if (rcStrict == VINF_IEM_RAISED_XCPT)
14211	{
14212	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14213	rcStrict = VINF_SUCCESS;
14214	}
14215
14216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14217	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14218	return rcStrict;
14219	}
14220
14221
14222	/**
14223	* VM-exit helper for CLTS.
14224	*/
14225	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
14226	{
14227	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14228	AssertRCReturn(rc, rc);
14229
14230	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
14231	AssertMsg( rcStrict == VINF_SUCCESS
14232	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14233
14234	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14235	if (rcStrict == VINF_IEM_RAISED_XCPT)
14236	{
14237	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14238	rcStrict = VINF_SUCCESS;
14239	}
14240
14241	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14242	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14243	return rcStrict;
14244	}
14245
14246
14247	/**
14248	* VM-exit helper for MOV from CRx (CRx read).
14249	*/
14250	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14251	{
14252	Assert(iCrReg < 16);
14253	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14254
14255	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14256	AssertRCReturn(rc, rc);
14257
14258	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
14259	AssertMsg( rcStrict == VINF_SUCCESS
14260	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14261
14262	if (iGReg == X86_GREG_xSP)
14263	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14264	else
14265	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14266	#ifdef VBOX_WITH_STATISTICS
14267	switch (iCrReg)
14268	{
14269	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14270	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14271	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14272	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14273	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14274	}
14275	#endif
14276	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
14277	return rcStrict;
14278	}
14279
14280
14281	/**
14282	* VM-exit helper for MOV to CRx (CRx write).
14283	*/
14284	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14285	{
14286	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14287	AssertRCReturn(rc, rc);
14288
14289	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
14290	AssertMsg( rcStrict == VINF_SUCCESS
14291	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14292	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14293
14294	switch (iCrReg)
14295	{
14296	case 0:
14297	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14299	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14300	break;
14301
14302	case 2:
14303	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14304	/* Nothing to do here, CR2 it's not part of the VMCS. */
14305	break;
14306
14307	case 3:
14308	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14310	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14311	break;
14312
14313	case 4:
14314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14316	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14317	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14318	break;
14319
14320	case 8:
14321	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14322	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14323	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14324	break;
14325
14326	default:
14327	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
14328	break;
14329	}
14330
14331	if (rcStrict == VINF_IEM_RAISED_XCPT)
14332	{
14333	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14334	rcStrict = VINF_SUCCESS;
14335	}
14336	return rcStrict;
14337	}
14338
14339
14340	/**
14341	* VM-exit exception handler for \#PF (Page-fault exception).
14342	*/
14343	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14344	{
14345	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14346	PVM pVM = pVCpu->CTX_SUFF(pVM);
14347	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14348	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14349	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14350	AssertRCReturn(rc, rc);
14351
14352	if (!pVM->hm.s.fNestedPaging)
14353	{ /* likely */ }
14354	else
14355	{
14356	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14357	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
14358	#endif
14359	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14360	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14361	{
14362	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14363	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
14364	}
14365	else
14366	{
14367	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14368	hmR0VmxSetPendingXcptDF(pVCpu);
14369	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
14370	}
14371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14372	return rc;
14373	}
14374
14375	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14376	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14377	if (pVmxTransient->fVectoringPF)
14378	{
14379	Assert(pVCpu->hm.s.Event.fPending);
14380	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14381	}
14382
14383	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14384	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14385	AssertRCReturn(rc, rc);
14386
14387	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
14388	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
14389
14390	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14391	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
14392
14393	Log4Func(("#PF: rc=%Rrc\n", rc));
14394	if (rc == VINF_SUCCESS)
14395	{
14396	/*
14397	* This is typically a shadow page table sync or a MMIO instruction. But we may have
14398	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14399	*/
14400	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14401	TRPMResetTrap(pVCpu);
14402	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14403	return rc;
14404	}
14405
14406	if (rc == VINF_EM_RAW_GUEST_TRAP)
14407	{
14408	if (!pVmxTransient->fVectoringDoublePF)
14409	{
14410	/* It's a guest page fault and needs to be reflected to the guest. */
14411	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14412	TRPMResetTrap(pVCpu);
14413	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14414	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14415	uGstErrorCode, pVmxTransient->uExitQual);
14416	}
14417	else
14418	{
14419	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14420	TRPMResetTrap(pVCpu);
14421	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14422	hmR0VmxSetPendingXcptDF(pVCpu);
14423	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
14424	}
14425
14426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14427	return VINF_SUCCESS;
14428	}
14429
14430	TRPMResetTrap(pVCpu);
14431	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14432	return rc;
14433	}
14434
14435
14436	/**
14437	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14438	*/
14439	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14440	{
14441	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14442	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14443
14444	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14445	AssertRCReturn(rc, rc);
14446
14447	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14448	{
14449	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14450	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14451
14452	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14453	* provides VM-exit instruction length. If this causes problem later,
14454	* disassemble the instruction like it's done on AMD-V. */
14455	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14456	AssertRCReturn(rc2, rc2);
14457	return rc;
14458	}
14459
14460	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14461	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14462	return rc;
14463	}
14464
14465
14466	/**
14467	* VM-exit exception handler for \#BP (Breakpoint exception).
14468	*/
14469	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14470	{
14471	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14473
14474	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14475	AssertRCReturn(rc, rc);
14476
14477	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14478	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14479	if (rc == VINF_EM_RAW_GUEST_TRAP)
14480	{
14481	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14482	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14483	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14484	AssertRCReturn(rc, rc);
14485
14486	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14487	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14488	}
14489
14490	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14491	return rc;
14492	}
14493
14494
14495	/**
14496	* VM-exit exception handler for \#AC (alignment check exception).
14497	*/
14498	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14499	{
14500	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14501	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
14502
14503	/*
14504	* Re-inject it. We'll detect any nesting before getting here.
14505	*/
14506	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14507	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14508	AssertRCReturn(rc, rc);
14509	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14510
14511	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14512	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14513	return VINF_SUCCESS;
14514	}
14515
14516
14517	/**
14518	* VM-exit exception handler for \#DB (Debug exception).
14519	*/
14520	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14521	{
14522	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14524
14525	/*
14526	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
14527	*/
14528	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14529
14530	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14531	uint64_t const uDR6 = X86_DR6_INIT_VAL
14532	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14533	\| X86_DR6_BD \| X86_DR6_BS));
14534
14535	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14536	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14537	Log6Func(("rc=%Rrc\n", rc));
14538	if (rc == VINF_EM_RAW_GUEST_TRAP)
14539	{
14540	/*
14541	* The exception was for the guest. Update DR6, DR7.GD and
14542	* IA32_DEBUGCTL.LBR before forwarding it.
14543	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14544	*/
14545	VMMRZCallRing3Disable(pVCpu);
14546	HM_DISABLE_PREEMPT(pVCpu);
14547
14548	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14549	pCtx->dr[6] \|= uDR6;
14550	if (CPUMIsGuestDebugStateActive(pVCpu))
14551	ASMSetDR6(pCtx->dr[6]);
14552
14553	HM_RESTORE_PREEMPT();
14554	VMMRZCallRing3Enable(pVCpu);
14555
14556	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14557	AssertRCReturn(rc, rc);
14558
14559	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14560	pCtx->dr[7] &= ~X86_DR7_GD;
14561
14562	/* Paranoia. */
14563	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14564	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14565
14566	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14567	AssertRCReturn(rc, rc);
14568
14569	/*
14570	* Raise #DB in the guest.
14571	*
14572	* It is important to reflect exactly what the VM-exit gave us (preserving the
14573	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14574	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14575	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14576	*
14577	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14578	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14579	*/
14580	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14581	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14582	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14583	AssertRCReturn(rc, rc);
14584	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14585	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14586	return VINF_SUCCESS;
14587	}
14588
14589	/*
14590	* Not a guest trap, must be a hypervisor related debug event then.
14591	* Update DR6 in case someone is interested in it.
14592	*/
14593	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14594	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14595	CPUMSetHyperDR6(pVCpu, uDR6);
14596
14597	return rc;
14598	}
14599
14600
14601	/**
14602	* Hacks its way around the lovely mesa driver's backdoor accesses.
14603	*
14604	* @sa hmR0SvmHandleMesaDrvGp.
14605	*/
14606	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14607	{
14608	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14609	RT_NOREF(pCtx);
14610
14611	/* For now we'll just skip the instruction. */
14612	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14613	}
14614
14615
14616	/**
14617	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14618	* backdoor logging w/o checking what it is running inside.
14619	*
14620	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14621	* backdoor port and magic numbers loaded in registers.
14622	*
14623	* @returns true if it is, false if it isn't.
14624	* @sa hmR0SvmIsMesaDrvGp.
14625	*/
14626	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14627	{
14628	/* 0xed: IN eAX,dx */
14629	uint8_t abInstr[1];
14630	if (pVmxTransient->cbInstr != sizeof(abInstr))
14631	return false;
14632
14633	/* Check that it is #GP(0). */
14634	if (pVmxTransient->uExitIntErrorCode != 0)
14635	return false;
14636
14637	/* Check magic and port. */
14638	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14639	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14640	if (pCtx->rax != UINT32_C(0x564d5868))
14641	return false;
14642	if (pCtx->dx != UINT32_C(0x5658))
14643	return false;
14644
14645	/* Flat ring-3 CS. */
14646	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14647	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14648	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14649	if (pCtx->cs.Attr.n.u2Dpl != 3)
14650	return false;
14651	if (pCtx->cs.u64Base != 0)
14652	return false;
14653
14654	/* Check opcode. */
14655	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14656	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14657	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14658	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14659	if (RT_FAILURE(rc))
14660	return false;
14661	if (abInstr[0] != 0xed)
14662	return false;
14663
14664	return true;
14665	}
14666
14667	/**
14668	* VM-exit exception handler for \#GP (General-protection exception).
14669	*
14670	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14671	*/
14672	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14673	{
14674	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14676
14677	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14678	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14679	if (pVmcsInfo->RealMode.fRealOnV86Active)
14680	{ /* likely */ }
14681	else
14682	{
14683	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14684	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14685	#endif
14686	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14687	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14688	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14689	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14690	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14691	AssertRCReturn(rc, rc);
14692	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14693	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14694
14695	if ( pVmxTransient->fIsNestedGuest
14696	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14697	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14698	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14699	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14700	else
14701	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14702	return rc;
14703	}
14704
14705	Assert(CPUMIsGuestInRealModeEx(pCtx));
14706	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14707	Assert(!pVmxTransient->fIsNestedGuest);
14708
14709	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14710	AssertRCReturn(rc, rc);
14711
14712	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14713	if (rcStrict == VINF_SUCCESS)
14714	{
14715	if (!CPUMIsGuestInRealModeEx(pCtx))
14716	{
14717	/*
14718	* The guest is no longer in real-mode, check if we can continue executing the
14719	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14720	*/
14721	pVmcsInfo->RealMode.fRealOnV86Active = false;
14722	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14723	{
14724	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14725	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14726	}
14727	else
14728	{
14729	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14730	rcStrict = VINF_EM_RESCHEDULE;
14731	}
14732	}
14733	else
14734	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14735	}
14736	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14737	{
14738	rcStrict = VINF_SUCCESS;
14739	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14740	}
14741	return VBOXSTRICTRC_VAL(rcStrict);
14742	}
14743
14744
14745	/**
14746	* VM-exit exception handler wrapper for generic exceptions.
14747	*
14748	* This simply re-injects the exception back into the VM without any special
14749	* processing.
14750	*/
14751	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14752	{
14753	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14754
14755	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14756	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14757	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14758	("uVector=%#x u32XcptBitmap=%#X32\n",
14759	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14760	NOREF(pVmcsInfo);
14761	#endif
14762
14763	/*
14764	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14765	* would have been handled while checking exits due to event delivery.
14766	*/
14767	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14768	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14769	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14770	AssertRCReturn(rc, rc);
14771
14772	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14773
14774	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14775	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14776	AssertRCReturn(rc, rc);
14777	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14778	#endif
14779
14780	#ifdef VBOX_WITH_STATISTICS
14781	switch (uVector)
14782	{
14783	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14784	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14785	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14786	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14787	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14788	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14789	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14790	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14791	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14792	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14793	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14794	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14795	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14796	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14797	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14798	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14799	default:
14800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14801	break;
14802	}
14803	#endif
14804
14805	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14806	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14807	NOREF(uVector);
14808
14809	/* Re-inject the original exception into the guest. */
14810	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14811	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14812	return VINF_SUCCESS;
14813	}
14814	/** @} */
14815
14816
14817	/** @name VM-exit handlers.
14818	* @{
14819	*/
14820	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14821	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14822	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14823
14824	/**
14825	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14826	*/
14827	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14828	{
14829	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14831	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14832	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14833	return VINF_SUCCESS;
14834	return VINF_EM_RAW_INTERRUPT;
14835	}
14836
14837
14838	/**
14839	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14840	* VM-exit.
14841	*/
14842	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14843	{
14844	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14845	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14846
14847	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14848	AssertRCReturn(rc, rc);
14849
14850	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14851	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14852
14853	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14854	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14855	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14856	NOREF(pVmcsInfo);
14857
14858	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
14859	{
14860	/*
14861	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14862	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14863	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14864	*
14865	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14866	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
14867	*/
14868	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14869	return hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14870	}
14871
14872	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14873	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14874	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
14875	{ /* likely */ }
14876	else
14877	{
14878	if (rcStrict == VINF_HM_DOUBLE_FAULT)
14879	{
14880	Assert(pVCpu->hm.s.Event.fPending);
14881	rcStrict = VINF_SUCCESS;
14882	}
14883	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14884	return rcStrict;
14885	}
14886
14887	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
14888	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
14889	switch (uIntType)
14890	{
14891	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
14892	Assert(uVector == X86_XCPT_DB);
14893	RT_FALL_THRU();
14894	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
14895	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14896	RT_FALL_THRU();
14897	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14898	{
14899	/*
14900	* If there's any exception caused as a result of event injection, the resulting
14901	* secondary/final execption will be pending, we shall continue guest execution
14902	* after injecting the event. The page-fault case is complicated and we manually
14903	* handle any currently pending event in hmR0VmxExitXcptPF.
14904	*/
14905	if (!pVCpu->hm.s.Event.fPending)
14906	{ /* likely */ }
14907	else if (uVector != X86_XCPT_PF)
14908	{
14909	rcStrict = VINF_SUCCESS;
14910	break;
14911	}
14912
14913	switch (uVector)
14914	{
14915	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
14916	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
14917	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
14918	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
14919	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
14920	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
14921	default:
14922	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient);
14923	break;
14924	}
14925	break;
14926	}
14927
14928	default:
14929	{
14930	pVCpu->hm.s.u32HMError = uExitIntInfo;
14931	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14932	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
14933	break;
14934	}
14935	}
14936	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14937	return rcStrict;
14938	}
14939
14940
14941	/**
14942	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14943	*/
14944	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14945	{
14946	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14947
14948	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14949	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14950	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14951	AssertRCReturn(rc, rc);
14952
14953	/* Evaluate and deliver pending events and resume guest execution. */
14954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14955	return VINF_SUCCESS;
14956	}
14957
14958
14959	/**
14960	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14961	*/
14962	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14963	{
14964	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14965
14966	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14967	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14968	{
14969	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14970	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14971	}
14972
14973	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14974
14975	/*
14976	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14977	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14978	*/
14979	uint32_t fIntrState;
14980	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14981	AssertRCReturn(rc, rc);
14982	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14983	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14984	{
14985	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14986	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14987
14988	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14989	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14990	AssertRCReturn(rc, rc);
14991	}
14992
14993	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14994	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14995	AssertRCReturn(rc, rc);
14996
14997	/* Evaluate and deliver pending events and resume guest execution. */
14998	return VINF_SUCCESS;
14999	}
15000
15001
15002	/**
15003	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
15004	*/
15005	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15006	{
15007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15008	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15009	}
15010
15011
15012	/**
15013	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
15014	*/
15015	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15016	{
15017	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15018	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15019	}
15020
15021
15022	/**
15023	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
15024	*/
15025	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15026	{
15027	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15028
15029	/*
15030	* Get the state we need and update the exit history entry.
15031	*/
15032	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15033	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15034	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15035	AssertRCReturn(rc, rc);
15036
15037	VBOXSTRICTRC rcStrict;
15038	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15039	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
15040	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15041	if (!pExitRec)
15042	{
15043	/*
15044	* Regular CPUID instruction execution.
15045	*/
15046	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
15047	if (rcStrict == VINF_SUCCESS)
15048	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15049	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15050	{
15051	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15052	rcStrict = VINF_SUCCESS;
15053	}
15054	}
15055	else
15056	{
15057	/*
15058	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15059	*/
15060	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15061	AssertRCReturn(rc2, rc2);
15062
15063	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
15064	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
15065
15066	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15067	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15068
15069	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15070	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15071	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15072	}
15073	return rcStrict;
15074	}
15075
15076
15077	/**
15078	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
15079	*/
15080	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15081	{
15082	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15083
15084	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15085	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
15086	AssertRCReturn(rc, rc);
15087
15088	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
15089	return VINF_EM_RAW_EMULATE_INSTR;
15090
15091	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
15092	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15093	}
15094
15095
15096	/**
15097	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
15098	*/
15099	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15100	{
15101	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15102
15103	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15104	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15105	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15106	AssertRCReturn(rc, rc);
15107
15108	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
15109	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15110	{
15111	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15112	we must reset offsetting on VM-entry. See @bugref{6634}. */
15113	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15114	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15115	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15116	}
15117	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15118	{
15119	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15120	rcStrict = VINF_SUCCESS;
15121	}
15122	return rcStrict;
15123	}
15124
15125
15126	/**
15127	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
15128	*/
15129	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15130	{
15131	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15132
15133	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15134	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
15135	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15136	AssertRCReturn(rc, rc);
15137
15138	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
15139	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15140	{
15141	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15142	we must reset offsetting on VM-reentry. See @bugref{6634}. */
15143	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15144	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15146	}
15147	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15148	{
15149	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15150	rcStrict = VINF_SUCCESS;
15151	}
15152	return rcStrict;
15153	}
15154
15155
15156	/**
15157	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15158	*/
15159	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15160	{
15161	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15162
15163	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15164	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
15165	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
15166	AssertRCReturn(rc, rc);
15167
15168	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15169	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15170	if (RT_LIKELY(rc == VINF_SUCCESS))
15171	{
15172	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15173	Assert(pVmxTransient->cbInstr == 2);
15174	}
15175	else
15176	{
15177	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
15178	rc = VERR_EM_INTERPRETER;
15179	}
15180	return rc;
15181	}
15182
15183
15184	/**
15185	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
15186	*/
15187	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15188	{
15189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15190
15191	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
15192	if (EMAreHypercallInstructionsEnabled(pVCpu))
15193	{
15194	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15195	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
15196	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
15197	AssertRCReturn(rc, rc);
15198
15199	/* Perform the hypercall. */
15200	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
15201	if (rcStrict == VINF_SUCCESS)
15202	{
15203	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15204	AssertRCReturn(rc, rc);
15205	}
15206	else
15207	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
15208	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
15209	\|\| RT_FAILURE(rcStrict));
15210
15211	/* If the hypercall changes anything other than guest's general-purpose registers,
15212	we would need to reload the guest changed bits here before VM-entry. */
15213	}
15214	else
15215	Log4Func(("Hypercalls not enabled\n"));
15216
15217	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
15218	if (RT_FAILURE(rcStrict))
15219	{
15220	hmR0VmxSetPendingXcptUD(pVCpu);
15221	rcStrict = VINF_SUCCESS;
15222	}
15223
15224	return rcStrict;
15225	}
15226
15227
15228	/**
15229	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15230	*/
15231	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15232	{
15233	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15234	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
15235
15236	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15237	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15238	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15239	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15240	AssertRCReturn(rc, rc);
15241
15242	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
15243
15244	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
15245	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15246	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15247	{
15248	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15249	rcStrict = VINF_SUCCESS;
15250	}
15251	else
15252	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
15253	VBOXSTRICTRC_VAL(rcStrict)));
15254	return rcStrict;
15255	}
15256
15257
15258	/**
15259	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
15260	*/
15261	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15262	{
15263	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15264
15265	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15266	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
15267	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15268	AssertRCReturn(rc, rc);
15269
15270	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
15271	if (rcStrict == VINF_SUCCESS)
15272	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15273	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15274	{
15275	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15276	rcStrict = VINF_SUCCESS;
15277	}
15278
15279	return rcStrict;
15280	}
15281
15282
15283	/**
15284	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
15285	*/
15286	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15287	{
15288	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15289
15290	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15291	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15292	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15293	AssertRCReturn(rc, rc);
15294
15295	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
15296	if (RT_SUCCESS(rcStrict))
15297	{
15298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15299	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
15300	rcStrict = VINF_SUCCESS;
15301	}
15302
15303	return rcStrict;
15304	}
15305
15306
15307	/**
15308	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
15309	* VM-exit.
15310	*/
15311	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15312	{
15313	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15314	return VINF_EM_RESET;
15315	}
15316
15317
15318	/**
15319	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15320	*/
15321	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15322	{
15323	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15324
15325	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15326	AssertRCReturn(rc, rc);
15327
15328	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
15329	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
15330	rc = VINF_SUCCESS;
15331	else
15332	rc = VINF_EM_HALT;
15333
15334	if (rc != VINF_SUCCESS)
15335	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
15336	return rc;
15337	}
15338
15339
15340	/**
15341	* VM-exit handler for instructions that result in a \#UD exception delivered to
15342	* the guest.
15343	*/
15344	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15345	{
15346	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15347	hmR0VmxSetPendingXcptUD(pVCpu);
15348	return VINF_SUCCESS;
15349	}
15350
15351
15352	/**
15353	* VM-exit handler for expiry of the VMX-preemption timer.
15354	*/
15355	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15356	{
15357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15358
15359	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
15360	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15361
15362	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
15363	PVM pVM = pVCpu->CTX_SUFF(pVM);
15364	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
15365	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
15366	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
15367	}
15368
15369
15370	/**
15371	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
15372	*/
15373	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15374	{
15375	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15376
15377	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15378	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15379	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
15380	AssertRCReturn(rc, rc);
15381
15382	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
15383	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15384	: HM_CHANGED_RAISED_XCPT_MASK);
15385
15386	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15387	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
15388
15389	return rcStrict;
15390	}
15391
15392
15393	/**
15394	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
15395	*/
15396	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15397	{
15398	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15399	/** @todo Use VM-exit instruction information. */
15400	return VERR_EM_INTERPRETER;
15401	}
15402
15403
15404	/**
15405	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
15406	* VM-exit.
15407	*/
15408	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15409	{
15410	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15411	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15412	AssertRCReturn(rc, rc);
15413
15414	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
15415	if (RT_FAILURE(rc))
15416	return rc;
15417
15418	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
15419	NOREF(uInvalidReason);
15420
15421	#ifdef VBOX_STRICT
15422	uint32_t fIntrState;
15423	RTHCUINTREG uHCReg;
15424	uint64_t u64Val;
15425	uint32_t u32Val;
15426	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
15427	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
15428	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
15429	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
15430	AssertRCReturn(rc, rc);
15431
15432	Log4(("uInvalidReason %u\n", uInvalidReason));
15433	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
15434	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
15435	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
15436	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
15437
15438	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
15439	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
15440	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
15441	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
15442	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
15443	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15444	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
15445	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
15446	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
15447	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15448	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
15449	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
15450
15451	hmR0DumpRegs(pVCpu);
15452	#endif
15453
15454	return VERR_VMX_INVALID_GUEST_STATE;
15455	}
15456
15457	/**
15458	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
15459	*/
15460	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15461	{
15462	/*
15463	* Cummulative notes of all recognized but unexpected VM-exits.
15464	*
15465	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
15466	* nested-paging is used.
15467	*
15468	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
15469	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
15470	* this function (and thereby stop VM execution) for handling such instructions.
15471	*
15472	*
15473	* VMX_EXIT_INIT_SIGNAL:
15474	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15475	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15476	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15477	*
15478	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15479	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15480	* See Intel spec. "23.8 Restrictions on VMX operation".
15481	*
15482	* VMX_EXIT_SIPI:
15483	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15484	* activity state is used. We don't make use of it as our guests don't have direct
15485	* access to the host local APIC.
15486	*
15487	* See Intel spec. 25.3 "Other Causes of VM-exits".
15488	*
15489	* VMX_EXIT_IO_SMI:
15490	* VMX_EXIT_SMI:
15491	* This can only happen if we support dual-monitor treatment of SMI, which can be
15492	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15493	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15494	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15495	*
15496	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15497	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15498	*
15499	* VMX_EXIT_ERR_MSR_LOAD:
15500	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15501	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15502	* execution.
15503	*
15504	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15505	*
15506	* VMX_EXIT_ERR_MACHINE_CHECK:
15507	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15508	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15509	* #MC exception abort class exception is raised. We thus cannot assume a
15510	* reasonable chance of continuing any sort of execution and we bail.
15511	*
15512	* See Intel spec. 15.1 "Machine-check Architecture".
15513	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15514	*
15515	* VMX_EXIT_PML_FULL:
15516	* VMX_EXIT_VIRTUALIZED_EOI:
15517	* VMX_EXIT_APIC_WRITE:
15518	* We do not currently support any of these features and thus they are all unexpected
15519	* VM-exits.
15520	*
15521	* VMX_EXIT_GDTR_IDTR_ACCESS:
15522	* VMX_EXIT_LDTR_TR_ACCESS:
15523	* VMX_EXIT_RDRAND:
15524	* VMX_EXIT_RSM:
15525	* VMX_EXIT_VMFUNC:
15526	* VMX_EXIT_ENCLS:
15527	* VMX_EXIT_RDSEED:
15528	* VMX_EXIT_XSAVES:
15529	* VMX_EXIT_XRSTORS:
15530	* VMX_EXIT_UMWAIT:
15531	* VMX_EXIT_TPAUSE:
15532	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15533	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15534	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15535	*
15536	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15537	*/
15538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15539	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15540	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15541	}
15542
15543
15544	/**
15545	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15546	*/
15547	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15548	{
15549	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15550
15551	/** @todo Optimize this: We currently drag in in the whole MSR state
15552	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15553	* MSRs required. That would require changes to IEM and possibly CPUM too.
15554	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15555	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15556	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15557	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15558	switch (idMsr)
15559	{
15560	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15561	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15562	}
15563
15564	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15565	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15566	AssertRCReturn(rc, rc);
15567
15568	Log4Func(("ecx=%#RX32\n", idMsr));
15569
15570	#ifdef VBOX_STRICT
15571	Assert(!pVmxTransient->fIsNestedGuest);
15572	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15573	{
15574	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15575	&& idMsr != MSR_K6_EFER)
15576	{
15577	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15578	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15579	}
15580	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15581	{
15582	Assert(pVmcsInfo->pvMsrBitmap);
15583	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15584	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15585	{
15586	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15587	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15588	}
15589	}
15590	}
15591	#endif
15592
15593	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15595	if (rcStrict == VINF_SUCCESS)
15596	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15597	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15598	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15599	{
15600	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15601	rcStrict = VINF_SUCCESS;
15602	}
15603	else
15604	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15605
15606	return rcStrict;
15607	}
15608
15609
15610	/**
15611	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15612	*/
15613	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15614	{
15615	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15616
15617	/** @todo Optimize this: We currently drag in in the whole MSR state
15618	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15619	* MSRs required. That would require changes to IEM and possibly CPUM too.
15620	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15621	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15622	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15623
15624	/*
15625	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15626	* Although we don't need to fetch the base as it will be overwritten shortly, while
15627	* loading guest-state we would also load the entire segment register including limit
15628	* and attributes and thus we need to load them here.
15629	*/
15630	switch (idMsr)
15631	{
15632	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15633	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15634	}
15635
15636	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15637	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15638	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15639	AssertRCReturn(rc, rc);
15640
15641	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15642
15643	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15644	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15645
15646	if (rcStrict == VINF_SUCCESS)
15647	{
15648	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15649
15650	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15651	if ( idMsr == MSR_IA32_APICBASE
15652	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15653	&& idMsr <= MSR_IA32_X2APIC_END))
15654	{
15655	/*
15656	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15657	* When full APIC register virtualization is implemented we'll have to make
15658	* sure APIC state is saved from the VMCS before IEM changes it.
15659	*/
15660	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15661	}
15662	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15663	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15664	else if (idMsr == MSR_K6_EFER)
15665	{
15666	/*
15667	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15668	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15669	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15670	*/
15671	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15672	}
15673
15674	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15675	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15676	{
15677	switch (idMsr)
15678	{
15679	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15680	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15681	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15682	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15683	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15684	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15685	default:
15686	{
15687	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15688	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15689	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15690	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15691	break;
15692	}
15693	}
15694	}
15695	#ifdef VBOX_STRICT
15696	else
15697	{
15698	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15699	switch (idMsr)
15700	{
15701	case MSR_IA32_SYSENTER_CS:
15702	case MSR_IA32_SYSENTER_EIP:
15703	case MSR_IA32_SYSENTER_ESP:
15704	case MSR_K8_FS_BASE:
15705	case MSR_K8_GS_BASE:
15706	{
15707	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15708	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15709	}
15710
15711	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15712	default:
15713	{
15714	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15715	{
15716	/* EFER MSR writes are always intercepted. */
15717	if (idMsr != MSR_K6_EFER)
15718	{
15719	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15720	idMsr));
15721	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15722	}
15723	}
15724
15725	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15726	{
15727	Assert(pVmcsInfo->pvMsrBitmap);
15728	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15729	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15730	{
15731	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15732	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15733	}
15734	}
15735	break;
15736	}
15737	}
15738	}
15739	#endif /* VBOX_STRICT */
15740	}
15741	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15742	{
15743	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15744	rcStrict = VINF_SUCCESS;
15745	}
15746	else
15747	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15748
15749	return rcStrict;
15750	}
15751
15752
15753	/**
15754	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15755	*/
15756	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15757	{
15758	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15759
15760	/** @todo The guest has likely hit a contended spinlock. We might want to
15761	* poke a schedule different guest VCPU. */
15762	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15763	if (RT_SUCCESS(rc))
15764	return VINF_EM_RAW_INTERRUPT;
15765
15766	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15767	return rc;
15768	}
15769
15770
15771	/**
15772	* VM-exit handler for when the TPR value is lowered below the specified
15773	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15774	*/
15775	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15776	{
15777	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15778	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15779
15780	/*
15781	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15782	* We'll re-evaluate pending interrupts and inject them before the next VM
15783	* entry so we can just continue execution here.
15784	*/
15785	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15786	return VINF_SUCCESS;
15787	}
15788
15789
15790	/**
15791	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15792	* VM-exit.
15793	*
15794	* @retval VINF_SUCCESS when guest execution can continue.
15795	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15796	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15797	* incompatible guest state for VMX execution (real-on-v86 case).
15798	*/
15799	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15800	{
15801	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15802	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15803
15804	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15805	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15806	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15807	AssertRCReturn(rc, rc);
15808
15809	VBOXSTRICTRC rcStrict;
15810	PVM pVM = pVCpu->CTX_SUFF(pVM);
15811	uint64_t const uExitQual = pVmxTransient->uExitQual;
15812	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15813	switch (uAccessType)
15814	{
15815	/*
15816	* MOV to CRx.
15817	*/
15818	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15819	{
15820	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15821	AssertRCReturn(rc, rc);
15822
15823	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15824	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15825	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15826	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15827
15828	/*
15829	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15830	* - When nested paging isn't used.
15831	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15832	* - We are executing in the VM debug loop.
15833	*/
15834	Assert( iCrReg != 3
15835	\|\| !pVM->hm.s.fNestedPaging
15836	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15837	\|\| pVCpu->hm.s.fUsingDebugLoop);
15838
15839	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15840	Assert( iCrReg != 8
15841	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15842
15843	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15844	AssertMsg( rcStrict == VINF_SUCCESS
15845	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15846
15847	/*
15848	* This is a kludge for handling switches back to real mode when we try to use
15849	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15850	* deal with special selector values, so we have to return to ring-3 and run
15851	* there till the selector values are V86 mode compatible.
15852	*
15853	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15854	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15855	* this function.
15856	*/
15857	if ( iCrReg == 0
15858	&& rcStrict == VINF_SUCCESS
15859	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15860	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15861	&& (uOldCr0 & X86_CR0_PE)
15862	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15863	{
15864	/** @todo Check selectors rather than returning all the time. */
15865	Assert(!pVmxTransient->fIsNestedGuest);
15866	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15867	rcStrict = VINF_EM_RESCHEDULE_REM;
15868	}
15869	break;
15870	}
15871
15872	/*
15873	* MOV from CRx.
15874	*/
15875	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15876	{
15877	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15878	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15879
15880	/*
15881	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15882	* - When nested paging isn't used.
15883	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15884	* - We are executing in the VM debug loop.
15885	*/
15886	Assert( iCrReg != 3
15887	\|\| !pVM->hm.s.fNestedPaging
15888	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15889	\|\| pVCpu->hm.s.fUsingDebugLoop);
15890
15891	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15892	Assert( iCrReg != 8
15893	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15894
15895	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15896	break;
15897	}
15898
15899	/*
15900	* CLTS (Clear Task-Switch Flag in CR0).
15901	*/
15902	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15903	{
15904	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15905	break;
15906	}
15907
15908	/*
15909	* LMSW (Load Machine-Status Word into CR0).
15910	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15911	*/
15912	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15913	{
15914	RTGCPTR GCPtrEffDst;
15915	uint8_t const cbInstr = pVmxTransient->cbInstr;
15916	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15917	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15918	if (fMemOperand)
15919	{
15920	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15921	AssertRCReturn(rc, rc);
15922	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15923	}
15924	else
15925	GCPtrEffDst = NIL_RTGCPTR;
15926	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15927	break;
15928	}
15929
15930	default:
15931	{
15932	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15933	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15934	}
15935	}
15936
15937	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15938	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15939	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15940
15941	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15942	NOREF(pVM);
15943	return rcStrict;
15944	}
15945
15946
15947	/**
15948	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15949	* VM-exit.
15950	*/
15951	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15952	{
15953	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15954	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15955
15956	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15957	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15958	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15959	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15960	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15961	\| CPUMCTX_EXTRN_EFER);
15962	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15963	AssertRCReturn(rc, rc);
15964
15965	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15966	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15967	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15968	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15969	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15970	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15971	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15972	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15973
15974	/*
15975	* Update exit history to see if this exit can be optimized.
15976	*/
15977	VBOXSTRICTRC rcStrict;
15978	PCEMEXITREC pExitRec = NULL;
15979	if ( !fGstStepping
15980	&& !fDbgStepping)
15981	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15982	!fIOString
15983	? !fIOWrite
15984	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15985	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15986	: !fIOWrite
15987	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15988	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15989	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15990	if (!pExitRec)
15991	{
15992	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15993	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15994
15995	uint32_t const cbValue = s_aIOSizes[uIOSize];
15996	uint32_t const cbInstr = pVmxTransient->cbInstr;
15997	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15998	PVM pVM = pVCpu->CTX_SUFF(pVM);
15999	if (fIOString)
16000	{
16001	/*
16002	* INS/OUTS - I/O String instruction.
16003	*
16004	* Use instruction-information if available, otherwise fall back on
16005	* interpreting the instruction.
16006	*/
16007	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
16008	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
16009	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
16010	if (fInsOutsInfo)
16011	{
16012	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16013	AssertRCReturn(rc2, rc2);
16014	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
16015	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
16016	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
16017	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
16018	if (fIOWrite)
16019	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
16020	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
16021	else
16022	{
16023	/*
16024	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
16025	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
16026	* See Intel Instruction spec. for "INS".
16027	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
16028	*/
16029	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
16030	}
16031	}
16032	else
16033	rcStrict = IEMExecOne(pVCpu);
16034
16035	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
16036	fUpdateRipAlready = true;
16037	}
16038	else
16039	{
16040	/*
16041	* IN/OUT - I/O instruction.
16042	*/
16043	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
16044	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
16045	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
16046	if (fIOWrite)
16047	{
16048	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
16049	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
16050	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
16051	&& !pCtx->eflags.Bits.u1TF)
16052	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
16053	}
16054	else
16055	{
16056	uint32_t u32Result = 0;
16057	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
16058	if (IOM_SUCCESS(rcStrict))
16059	{
16060	/* Save result of I/O IN instr. in AL/AX/EAX. */
16061	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
16062	}
16063	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
16064	&& !pCtx->eflags.Bits.u1TF)
16065	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
16066	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
16067	}
16068	}
16069
16070	if (IOM_SUCCESS(rcStrict))
16071	{
16072	if (!fUpdateRipAlready)
16073	{
16074	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
16075	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
16076	}
16077
16078	/*
16079	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
16080	* while booting Fedora 17 64-bit guest.
16081	*
16082	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
16083	*/
16084	if (fIOString)
16085	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
16086
16087	/*
16088	* If any I/O breakpoints are armed, we need to check if one triggered
16089	* and take appropriate action.
16090	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
16091	*/
16092	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
16093	AssertRCReturn(rc, rc);
16094
16095	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
16096	* execution engines about whether hyper BPs and such are pending. */
16097	uint32_t const uDr7 = pCtx->dr[7];
16098	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
16099	&& X86_DR7_ANY_RW_IO(uDr7)
16100	&& (pCtx->cr4 & X86_CR4_DE))
16101	\|\| DBGFBpIsHwIoArmed(pVM)))
16102	{
16103	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
16104
16105	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
16106	VMMRZCallRing3Disable(pVCpu);
16107	HM_DISABLE_PREEMPT(pVCpu);
16108
16109	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
16110
16111	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
16112	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
16113	{
16114	/* Raise #DB. */
16115	if (fIsGuestDbgActive)
16116	ASMSetDR6(pCtx->dr[6]);
16117	if (pCtx->dr[7] != uDr7)
16118	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
16119
16120	hmR0VmxSetPendingXcptDB(pVCpu);
16121	}
16122	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
16123	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
16124	else if ( rcStrict2 != VINF_SUCCESS
16125	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
16126	rcStrict = rcStrict2;
16127	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
16128
16129	HM_RESTORE_PREEMPT();
16130	VMMRZCallRing3Enable(pVCpu);
16131	}
16132	}
16133
16134	#ifdef VBOX_STRICT
16135	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
16136	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
16137	Assert(!fIOWrite);
16138	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
16139	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
16140	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
16141	Assert(fIOWrite);
16142	else
16143	{
16144	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
16145	* statuses, that the VMM device and some others may return. See
16146	* IOM_SUCCESS() for guidance. */
16147	AssertMsg( RT_FAILURE(rcStrict)
16148	\|\| rcStrict == VINF_SUCCESS
16149	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
16150	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
16151	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
16152	\|\| rcStrict == VINF_EM_RAW_TO_R3
16153	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16154	# endif
16155	}
16156	#endif
16157	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
16158	}
16159	else
16160	{
16161	/*
16162	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16163	*/
16164	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16165	AssertRCReturn(rc2, rc2);
16166	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
16167	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
16168	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
16169	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16170	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
16171	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
16172
16173	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16174	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16175
16176	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16177	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16178	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16179	}
16180	return rcStrict;
16181	}
16182
16183
16184	/**
16185	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
16186	* VM-exit.
16187	*/
16188	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16189	{
16190	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16191
16192	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
16193	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16194	AssertRCReturn(rc, rc);
16195	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
16196	{
16197	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16198	AssertRCReturn(rc, rc);
16199	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16200	{
16201	uint32_t uErrCode;
16202	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
16203	{
16204	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16205	AssertRCReturn(rc, rc);
16206	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
16207	}
16208	else
16209	uErrCode = 0;
16210
16211	RTGCUINTPTR GCPtrFaultAddress;
16212	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
16213	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
16214	else
16215	GCPtrFaultAddress = 0;
16216
16217	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16218	AssertRCReturn(rc, rc);
16219
16220	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
16221	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
16222
16223	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
16224	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
16225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16226	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16227	}
16228	}
16229
16230	/* Fall back to the interpreter to emulate the task-switch. */
16231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16232	return VERR_EM_INTERPRETER;
16233	}
16234
16235
16236	/**
16237	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
16238	*/
16239	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16240	{
16241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16242
16243	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16244	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
16245	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16246	AssertRCReturn(rc, rc);
16247	return VINF_EM_DBG_STEPPED;
16248	}
16249
16250
16251	/**
16252	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
16253	*/
16254	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16255	{
16256	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16257	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
16258
16259	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16260	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16261	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16262	{
16263	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
16264	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16265	{
16266	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16267	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16268	}
16269	}
16270	else
16271	{
16272	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16273	{
16274	Assert(pVCpu->hm.s.Event.fPending);
16275	rcStrict1 = VINF_SUCCESS;
16276	}
16277	return rcStrict1;
16278	}
16279
16280	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
16281	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16282	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16283	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16284	AssertRCReturn(rc, rc);
16285
16286	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
16287	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
16288	VBOXSTRICTRC rcStrict2;
16289	switch (uAccessType)
16290	{
16291	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
16292	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
16293	{
16294	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
16295	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
16296	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
16297
16298	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
16299	GCPhys &= PAGE_BASE_GC_MASK;
16300	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
16301	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
16302	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
16303
16304	PVM pVM = pVCpu->CTX_SUFF(pVM);
16305	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16306	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
16307	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
16308	CPUMCTX2CORE(pCtx), GCPhys);
16309	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16310	if ( rcStrict2 == VINF_SUCCESS
16311	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16312	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16313	{
16314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16315	\| HM_CHANGED_GUEST_APIC_TPR);
16316	rcStrict2 = VINF_SUCCESS;
16317	}
16318	break;
16319	}
16320
16321	default:
16322	{
16323	Log4Func(("uAccessType=%#x\n", uAccessType));
16324	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
16325	break;
16326	}
16327	}
16328
16329	if (rcStrict2 != VINF_SUCCESS)
16330	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
16331	return rcStrict2;
16332	}
16333
16334
16335	/**
16336	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
16337	* VM-exit.
16338	*/
16339	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16340	{
16341	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16342
16343	/* We should -not- get this VM-exit if the guest's debug registers were active. */
16344	if (pVmxTransient->fWasGuestDebugStateActive)
16345	{
16346	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
16347	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
16348	}
16349
16350	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16351	if ( !pVCpu->hm.s.fSingleInstruction
16352	&& !pVmxTransient->fWasHyperDebugStateActive)
16353	{
16354	Assert(!DBGFIsStepping(pVCpu));
16355	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
16356
16357	/* Don't intercept MOV DRx any more. */
16358	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
16359	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16360	AssertRCReturn(rc, rc);
16361
16362	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
16363	VMMRZCallRing3Disable(pVCpu);
16364	HM_DISABLE_PREEMPT(pVCpu);
16365
16366	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
16367	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
16368	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
16369
16370	HM_RESTORE_PREEMPT();
16371	VMMRZCallRing3Enable(pVCpu);
16372
16373	#ifdef VBOX_WITH_STATISTICS
16374	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16375	AssertRCReturn(rc, rc);
16376	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16378	else
16379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16380	#endif
16381	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
16382	return VINF_SUCCESS;
16383	}
16384
16385	/*
16386	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
16387	* The EFER MSR is always up-to-date.
16388	* Update the segment registers and DR7 from the CPU.
16389	*/
16390	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16391	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16392	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
16393	AssertRCReturn(rc, rc);
16394	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
16395
16396	PVM pVM = pVCpu->CTX_SUFF(pVM);
16397	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16398	{
16399	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16400	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
16401	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
16402	if (RT_SUCCESS(rc))
16403	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
16404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16405	}
16406	else
16407	{
16408	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16409	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
16410	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
16411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16412	}
16413
16414	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
16415	if (RT_SUCCESS(rc))
16416	{
16417	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
16418	AssertRCReturn(rc2, rc2);
16419	return VINF_SUCCESS;
16420	}
16421	return rc;
16422	}
16423
16424
16425	/**
16426	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
16427	* Conditional VM-exit.
16428	*/
16429	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16430	{
16431	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16432	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16433
16434	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16435	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16436	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16437	{
16438	/*
16439	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
16440	* instruction emulation to inject the original event. Otherwise, injecting the original event
16441	* using hardware-assisted VMX would would trigger the same EPT misconfig VM-exit again.
16442	*/
16443	if (!pVCpu->hm.s.Event.fPending)
16444	{ /* likely */ }
16445	else
16446	{
16447	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16448	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16449	/** @todo NSTVMX: Think about how this should be handled. */
16450	if (pVmxTransient->fIsNestedGuest)
16451	return VERR_VMX_IPE_3;
16452	#endif
16453	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16454	}
16455	}
16456	else
16457	{
16458	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16459	rcStrict = VINF_SUCCESS;
16460	return rcStrict;
16461	}
16462
16463	/*
16464	* Get sufficent state and update the exit history entry.
16465	*/
16466	RTGCPHYS GCPhys;
16467	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16468	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16469	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16470	AssertRCReturn(rc, rc);
16471
16472	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16473	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16474	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16475	if (!pExitRec)
16476	{
16477	/*
16478	* If we succeed, resume guest execution.
16479	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16480	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16481	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16482	* weird case. See @bugref{6043}.
16483	*/
16484	PVM pVM = pVCpu->CTX_SUFF(pVM);
16485	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16486	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16487	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16488	if ( rcStrict == VINF_SUCCESS
16489	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16490	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16491	{
16492	/* Successfully handled MMIO operation. */
16493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16494	\| HM_CHANGED_GUEST_APIC_TPR);
16495	rcStrict = VINF_SUCCESS;
16496	}
16497	}
16498	else
16499	{
16500	/*
16501	* Frequent exit or something needing probing. Call EMHistoryExec.
16502	*/
16503	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16504	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16505
16506	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16507	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16508
16509	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16510	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16511	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16512	}
16513	return rcStrict;
16514	}
16515
16516
16517	/**
16518	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16519	* VM-exit.
16520	*/
16521	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16522	{
16523	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16524	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16525
16526	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16527	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16528	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16529	{
16530	/*
16531	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16532	* we shall resolve the nested #PF and re-inject the original event.
16533	*/
16534	if (pVCpu->hm.s.Event.fPending)
16535	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16536	}
16537	else
16538	{
16539	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16540	{
16541	Assert(pVCpu->hm.s.Event.fPending);
16542	rcStrict = VINF_SUCCESS;
16543	}
16544	return rcStrict;
16545	}
16546
16547	RTGCPHYS GCPhys;
16548	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16549	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16550	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16551	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16552	AssertRCReturn(rc, rc);
16553
16554	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
16555	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
16556
16557	RTGCUINT uErrorCode = 0;
16558	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16559	uErrorCode \|= X86_TRAP_PF_ID;
16560	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16561	uErrorCode \|= X86_TRAP_PF_RW;
16562	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16563	uErrorCode \|= X86_TRAP_PF_P;
16564
16565	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16566
16567	/* Handle the pagefault trap for the nested shadow table. */
16568	PVM pVM = pVCpu->CTX_SUFF(pVM);
16569	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16570
16571	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
16572	pCtx->cs.Sel, pCtx->rip));
16573
16574	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16575	TRPMResetTrap(pVCpu);
16576
16577	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16578	if ( rcStrict == VINF_SUCCESS
16579	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16580	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16581	{
16582	/* Successfully synced our nested page tables. */
16583	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16584	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16585	return VINF_SUCCESS;
16586	}
16587
16588	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16589	return rcStrict;
16590	}
16591
16592
16593	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16594	/**
16595	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16596	*/
16597	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16598	{
16599	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16600
16601	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16602	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16603	\| CPUMCTX_EXTRN_HWVIRT
16604	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16605	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16606	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16607	AssertRCReturn(rc, rc);
16608
16609	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16610
16611	VMXVEXITINFO ExitInfo;
16612	RT_ZERO(ExitInfo);
16613	ExitInfo.uReason = pVmxTransient->uExitReason;
16614	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16615	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16616	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16617	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16618
16619	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16620	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16621	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16622	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16623	{
16624	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16625	rcStrict = VINF_SUCCESS;
16626	}
16627	return rcStrict;
16628	}
16629
16630
16631	/**
16632	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16633	*/
16634	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16635	{
16636	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16637
16638	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16639	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16640	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16641	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16642	AssertRCReturn(rc, rc);
16643
16644	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16645
16646	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16647	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16648	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16649	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16650	{
16651	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16652	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16653	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16654	}
16655	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16656	return rcStrict;
16657	}
16658
16659
16660	/**
16661	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16662	*/
16663	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16664	{
16665	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16666
16667	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16668	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16669	\| CPUMCTX_EXTRN_HWVIRT
16670	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16671	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16672	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16673	AssertRCReturn(rc, rc);
16674
16675	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16676
16677	VMXVEXITINFO ExitInfo;
16678	RT_ZERO(ExitInfo);
16679	ExitInfo.uReason = pVmxTransient->uExitReason;
16680	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16681	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16682	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16683	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16684
16685	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16686	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16687	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16688	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16689	{
16690	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16691	rcStrict = VINF_SUCCESS;
16692	}
16693	return rcStrict;
16694	}
16695
16696
16697	/**
16698	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16699	*/
16700	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16701	{
16702	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16703
16704	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16705	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16706	\| CPUMCTX_EXTRN_HWVIRT
16707	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16708	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16709	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16710	AssertRCReturn(rc, rc);
16711
16712	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16713
16714	VMXVEXITINFO ExitInfo;
16715	RT_ZERO(ExitInfo);
16716	ExitInfo.uReason = pVmxTransient->uExitReason;
16717	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16718	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16719	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16720	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16721
16722	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16723	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16724	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16725	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16726	{
16727	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16728	rcStrict = VINF_SUCCESS;
16729	}
16730	return rcStrict;
16731	}
16732
16733
16734	/**
16735	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16736	*/
16737	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16738	{
16739	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16740
16741	/*
16742	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16743	* thus might not need to import the shadow VMCS state, it's safer just in case
16744	* code elsewhere dares look at unsynced VMCS fields.
16745	*/
16746	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16747	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16748	\| CPUMCTX_EXTRN_HWVIRT
16749	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16750	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16751	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16752	AssertRCReturn(rc, rc);
16753
16754	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16755
16756	VMXVEXITINFO ExitInfo;
16757	RT_ZERO(ExitInfo);
16758	ExitInfo.uReason = pVmxTransient->uExitReason;
16759	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16760	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16761	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16762	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16763	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16764
16765	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16766	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16767	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16768	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16769	{
16770	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16771	rcStrict = VINF_SUCCESS;
16772	}
16773	return rcStrict;
16774	}
16775
16776
16777	/**
16778	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16779	*/
16780	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16781	{
16782	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16783
16784	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16785	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16786	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16787	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16788	AssertRCReturn(rc, rc);
16789
16790	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16791
16792	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16793	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16794	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16795	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16796	{
16797	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16798	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16799	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16800	}
16801	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16802	return rcStrict;
16803	}
16804
16805
16806	/**
16807	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16808	*/
16809	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16810	{
16811	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16812
16813	/*
16814	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16815	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16816	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16817	*/
16818	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16819	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16820	\| CPUMCTX_EXTRN_HWVIRT
16821	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16822	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16823	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16824	AssertRCReturn(rc, rc);
16825
16826	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16827
16828	VMXVEXITINFO ExitInfo;
16829	RT_ZERO(ExitInfo);
16830	ExitInfo.uReason = pVmxTransient->uExitReason;
16831	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16832	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16833	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16834	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16835	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16836
16837	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16838	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16839	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16840	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16841	{
16842	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16843	rcStrict = VINF_SUCCESS;
16844	}
16845	return rcStrict;
16846	}
16847
16848
16849	/**
16850	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16851	*/
16852	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16853	{
16854	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16855
16856	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16857	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16858	\| CPUMCTX_EXTRN_HWVIRT
16859	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16860	AssertRCReturn(rc, rc);
16861
16862	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16863
16864	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16865	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16866	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16867	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16868	{
16869	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16870	rcStrict = VINF_SUCCESS;
16871	}
16872	return rcStrict;
16873	}
16874
16875
16876	/**
16877	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16878	*/
16879	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16880	{
16881	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16882
16883	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16884	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16885	\| CPUMCTX_EXTRN_HWVIRT
16886	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16887	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16888	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16889	AssertRCReturn(rc, rc);
16890
16891	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16892
16893	VMXVEXITINFO ExitInfo;
16894	RT_ZERO(ExitInfo);
16895	ExitInfo.uReason = pVmxTransient->uExitReason;
16896	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16897	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16898	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16899	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16900
16901	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16902	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16903	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16904	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16905	{
16906	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16907	rcStrict = VINF_SUCCESS;
16908	}
16909	return rcStrict;
16910	}
16911
16912
16913	/**
16914	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16915	*/
16916	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16917	{
16918	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16919
16920	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16921	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16922	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16923	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16924	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16925	AssertRCReturn(rc, rc);
16926
16927	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16928
16929	VMXVEXITINFO ExitInfo;
16930	RT_ZERO(ExitInfo);
16931	ExitInfo.uReason = pVmxTransient->uExitReason;
16932	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16933	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16934	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16935	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16936
16937	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16938	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16939	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16940	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16941	{
16942	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16943	rcStrict = VINF_SUCCESS;
16944	}
16945	return rcStrict;
16946	}
16947	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16948	/** @} */
16949
16950
16951	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16952	/** @name Nested-guest VM-exit handlers.
16953	* @{
16954	*/
16955	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16956	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16957	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16958
16959	/**
16960	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16961	* Conditional VM-exit.
16962	*/
16963	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16964	{
16965	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16966
16967	int rc = hmR0VmxCheckExitDueToEventDeliveryNested(pVCpu, pVmxTransient);
16968	AssertRCReturn(rc, rc);
16969
16970	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16971	AssertRCReturn(rc, rc);
16972
16973	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16974	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16975	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16976
16977	switch (uExitIntType)
16978	{
16979	/*
16980	* Physical NMIs:
16981	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16982	*/
16983	case VMX_EXIT_INT_INFO_TYPE_NMI:
16984	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16985
16986	/*
16987	* Hardware exceptions,
16988	* Software exceptions,
16989	* Privileged software exceptions:
16990	* Figure out if the exception must be delivered to the guest or the nested-guest.
16991	*
16992	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16993	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16994	* length. However, if delivery of a software interrupt, software exception or privileged
16995	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16996	* Hence, we read it for all exception types below to keep it simple.
16997	*/
16998	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16999	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
17000	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
17001	{
17002	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
17003	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17004	AssertRCReturn(rc, rc);
17005
17006	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
17007	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
17008
17009	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
17010	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
17011	pVmxTransient->uExitIntErrorCode);
17012	if (fIntercept)
17013	{
17014	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17015	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17016	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17017	AssertRCReturn(rc, rc);
17018
17019	VMXVEXITINFO ExitInfo;
17020	RT_ZERO(ExitInfo);
17021	ExitInfo.uReason = pVmxTransient->uExitReason;
17022	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17023	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17024
17025	VMXVEXITEVENTINFO ExitEventInfo;
17026	RT_ZERO(ExitEventInfo);
17027	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
17028	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
17029	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17030	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17031	if (pVCpu->hm.s.Event.fPending)
17032	{
17033	Assert(ExitEventInfo.uIdtVectoringInfo == pVCpu->hm.s.Event.u64IntInfo);
17034	Assert(ExitEventInfo.uIdtVectoringErrCode == pVCpu->hm.s.Event.u32ErrCode);
17035	pVCpu->hm.s.Event.fPending = false;
17036	}
17037	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
17038	}
17039
17040	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
17041	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
17042
17043	/*
17044	* If the guest hypervisor is not intercepting an exception that caused a VM-exit directly,
17045	* forward it to the guest (for e.g, an instruction raises a #GP that causes a VM-exit but
17046	* the guest hypervisor is not intercept #GPs, inject #GP into the guest).
17047	*/
17048	if (!pVCpu->hm.s.Event.fPending)
17049	{
17050	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
17051	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
17052	}
17053	return VINF_SUCCESS;
17054	}
17055
17056	/*
17057	* Software interrupts:
17058	* VM-exits cannot be caused by software interrupts.
17059	*
17060	* External interrupts:
17061	* This should only happen when "acknowledge external interrupts on VM-exit"
17062	* control is set. However, we never set this when executing a guest or
17063	* nested-guest. For nested-guests it is emulated while injecting interrupts into
17064	* the guest.
17065	*/
17066	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
17067	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
17068	default:
17069	{
17070	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
17071	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
17072	}
17073	}
17074	}
17075
17076
17077	/**
17078	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
17079	* Unconditional VM-exit.
17080	*/
17081	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17082	{
17083	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17084	return IEMExecVmxVmexitTripleFault(pVCpu);
17085	}
17086
17087
17088	/**
17089	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
17090	*/
17091	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17092	{
17093	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17094
17095	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
17096	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17097	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
17098	}
17099
17100
17101	/**
17102	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
17103	*/
17104	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17105	{
17106	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17107
17108	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
17109	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17110	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
17111	}
17112
17113
17114	/**
17115	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
17116	* Unconditional VM-exit.
17117	*/
17118	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17119	{
17120	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17121
17122	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17123	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17124	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17125	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17126	AssertRCReturn(rc, rc);
17127
17128	VMXVEXITINFO ExitInfo;
17129	RT_ZERO(ExitInfo);
17130	ExitInfo.uReason = pVmxTransient->uExitReason;
17131	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17132	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17133
17134	VMXVEXITEVENTINFO ExitEventInfo;
17135	RT_ZERO(ExitEventInfo);
17136	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17137	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17138	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
17139	}
17140
17141
17142	/**
17143	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
17144	*/
17145	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17146	{
17147	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17148
17149	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
17150	{
17151	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17152	AssertRCReturn(rc, rc);
17153	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17154	}
17155	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
17156	}
17157
17158
17159	/**
17160	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
17161	*/
17162	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17163	{
17164	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17165
17166	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17167	{
17168	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17169	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17170	AssertRCReturn(rc, rc);
17171
17172	VMXVEXITINFO ExitInfo;
17173	RT_ZERO(ExitInfo);
17174	ExitInfo.uReason = pVmxTransient->uExitReason;
17175	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17176	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17177	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17178	}
17179	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
17180	}
17181
17182
17183	/**
17184	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
17185	*/
17186	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17187	{
17188	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17189
17190	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
17191	{
17192	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17193	AssertRCReturn(rc, rc);
17194	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17195	}
17196	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
17197	}
17198
17199
17200	/**
17201	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
17202	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
17203	*/
17204	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17205	{
17206	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17207
17208	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
17209	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
17210
17211	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17212	AssertRCReturn(rc, rc);
17213
17214	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
17215	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17216	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17217
17218	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
17219	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
17220	u64VmcsField &= UINT64_C(0xffffffff);
17221
17222	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
17223	{
17224	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17225	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17226	AssertRCReturn(rc, rc);
17227
17228	VMXVEXITINFO ExitInfo;
17229	RT_ZERO(ExitInfo);
17230	ExitInfo.uReason = pVmxTransient->uExitReason;
17231	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17232	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17233	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17234	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17235	}
17236
17237	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
17238	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
17239	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
17240	}
17241
17242
17243	/**
17244	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
17245	*/
17246	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17247	{
17248	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17249
17250	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17251	{
17252	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17253	AssertRCReturn(rc, rc);
17254	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17255	}
17256
17257	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
17258	}
17259
17260
17261	/**
17262	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
17263	* Conditional VM-exit.
17264	*/
17265	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17266	{
17267	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17268
17269	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17270	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17271	AssertRCReturn(rc, rc);
17272
17273	VBOXSTRICTRC rcStrict;
17274	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
17275	switch (uAccessType)
17276	{
17277	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
17278	{
17279	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17280	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17281	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17282	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17283
17284	bool fIntercept;
17285	switch (iCrReg)
17286	{
17287	case 0:
17288	case 4:
17289	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
17290	break;
17291
17292	case 3:
17293	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
17294	break;
17295
17296	case 8:
17297	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
17298	break;
17299
17300	default:
17301	fIntercept = false;
17302	break;
17303	}
17304	if (fIntercept)
17305	{
17306	VMXVEXITINFO ExitInfo;
17307	RT_ZERO(ExitInfo);
17308	ExitInfo.uReason = pVmxTransient->uExitReason;
17309	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17310	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17311	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17312	}
17313	else
17314	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17315	break;
17316	}
17317
17318	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
17319	{
17320	/*
17321	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
17322	* CR2 reads do not cause a VM-exit.
17323	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
17324	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
17325	*/
17326	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17327	if ( iCrReg == 3
17328	\|\| iCrReg == 8)
17329	{
17330	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
17331	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
17332	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
17333	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
17334	{
17335	VMXVEXITINFO ExitInfo;
17336	RT_ZERO(ExitInfo);
17337	ExitInfo.uReason = pVmxTransient->uExitReason;
17338	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17339	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17340	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17341	}
17342	else
17343	{
17344	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17345	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17346	}
17347	}
17348	else
17349	{
17350	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
17351	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
17352	}
17353	break;
17354	}
17355
17356	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
17357	{
17358	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
17359	Assert(pVmcsNstGst);
17360	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
17361	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
17362	if ( (uGstHostMask & X86_CR0_TS)
17363	&& (uReadShadow & X86_CR0_TS))
17364	{
17365	VMXVEXITINFO ExitInfo;
17366	RT_ZERO(ExitInfo);
17367	ExitInfo.uReason = pVmxTransient->uExitReason;
17368	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17369	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17370	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17371	}
17372	else
17373	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
17374	break;
17375	}
17376
17377	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
17378	{
17379	RTGCPTR GCPtrEffDst;
17380	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
17381	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
17382	if (fMemOperand)
17383	{
17384	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17385	AssertRCReturn(rc, rc);
17386	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
17387	}
17388	else
17389	GCPtrEffDst = NIL_RTGCPTR;
17390
17391	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
17392	{
17393	VMXVEXITINFO ExitInfo;
17394	RT_ZERO(ExitInfo);
17395	ExitInfo.uReason = pVmxTransient->uExitReason;
17396	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17397	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
17398	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17399	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17400	}
17401	else
17402	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
17403	break;
17404	}
17405
17406	default:
17407	{
17408	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
17409	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
17410	}
17411	}
17412
17413	if (rcStrict == VINF_IEM_RAISED_XCPT)
17414	{
17415	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
17416	rcStrict = VINF_SUCCESS;
17417	}
17418	return rcStrict;
17419	}
17420
17421
17422	/**
17423	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
17424	* Conditional VM-exit.
17425	*/
17426	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17427	{
17428	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17429
17430	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
17431	{
17432	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17433	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17434	AssertRCReturn(rc, rc);
17435
17436	VMXVEXITINFO ExitInfo;
17437	RT_ZERO(ExitInfo);
17438	ExitInfo.uReason = pVmxTransient->uExitReason;
17439	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17440	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17441	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17442	}
17443	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
17444	}
17445
17446
17447	/**
17448	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
17449	* Conditional VM-exit.
17450	*/
17451	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17452	{
17453	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17454
17455	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17456	AssertRCReturn(rc, rc);
17457
17458	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
17459	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
17460	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
17461
17462	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
17463	uint8_t const cbAccess = s_aIOSizes[uIOSize];
17464	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
17465	{
17466	/*
17467	* IN/OUT instruction:
17468	* - Provides VM-exit instruction length.
17469	*
17470	* INS/OUTS instruction:
17471	* - Provides VM-exit instruction length.
17472	* - Provides Guest-linear address.
17473	* - Optionally provides VM-exit instruction info (depends on CPU feature).
17474	*/
17475	PVM pVM = pVCpu->CTX_SUFF(pVM);
17476	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17477	AssertRCReturn(rc, rc);
17478
17479	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17480	pVmxTransient->ExitInstrInfo.u = 0;
17481	pVmxTransient->uGuestLinearAddr = 0;
17482
17483	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17484	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17485	if (fIOString)
17486	{
17487	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17488	if (fVmxInsOutsInfo)
17489	{
17490	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17491	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17492	}
17493	}
17494	AssertRCReturn(rc, rc);
17495
17496	VMXVEXITINFO ExitInfo;
17497	RT_ZERO(ExitInfo);
17498	ExitInfo.uReason = pVmxTransient->uExitReason;
17499	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17500	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17501	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17502	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17503	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17504	}
17505	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17506	}
17507
17508
17509	/**
17510	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17511	*/
17512	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17513	{
17514	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17515
17516	uint32_t fMsrpm;
17517	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17518	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17519	else
17520	fMsrpm = VMXMSRPM_EXIT_RD;
17521
17522	if (fMsrpm & VMXMSRPM_EXIT_RD)
17523	{
17524	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17525	AssertRCReturn(rc, rc);
17526	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17527	}
17528	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17529	}
17530
17531
17532	/**
17533	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17534	*/
17535	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17536	{
17537	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17538
17539	uint32_t fMsrpm;
17540	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17541	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17542	else
17543	fMsrpm = VMXMSRPM_EXIT_WR;
17544
17545	if (fMsrpm & VMXMSRPM_EXIT_WR)
17546	{
17547	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17548	AssertRCReturn(rc, rc);
17549	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17550	}
17551	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17552	}
17553
17554
17555	/**
17556	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17557	*/
17558	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17559	{
17560	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17561
17562	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17563	{
17564	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17565	AssertRCReturn(rc, rc);
17566	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17567	}
17568	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17569	}
17570
17571
17572	/**
17573	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17574	* VM-exit.
17575	*/
17576	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17577	{
17578	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17579
17580	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17581	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17582	}
17583
17584
17585	/**
17586	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17587	*/
17588	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17589	{
17590	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17591
17592	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17593	{
17594	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17595	AssertRCReturn(rc, rc);
17596	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17597	}
17598	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17599	}
17600
17601
17602	/**
17603	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17604	*/
17605	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17606	{
17607	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17608
17609	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17610	* PAUSE when executing a nested-guest? If it does not, we would not need
17611	* to check for the intercepts here. Just call VM-exit... */
17612
17613	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17614	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17615	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17616	{
17617	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17618	AssertRCReturn(rc, rc);
17619	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17620	}
17621	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17622	}
17623
17624
17625	/**
17626	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17627	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17628	*/
17629	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17630	{
17631	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17632
17633	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17634	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17635	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17636	}
17637
17638
17639	/**
17640	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17641	* VM-exit.
17642	*/
17643	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17644	{
17645	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17646
17647	int rc = hmR0VmxCheckExitDueToEventDeliveryNested(pVCpu, pVmxTransient);
17648	AssertRCReturn(rc, rc);
17649
17650	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17651	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17652	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17653	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17654	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17655	AssertRCReturn(rc, rc);
17656
17657	VMXVEXITINFO ExitInfo;
17658	RT_ZERO(ExitInfo);
17659	ExitInfo.uReason = pVmxTransient->uExitReason;
17660	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17661	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17662
17663	VMXVEXITEVENTINFO ExitEventInfo;
17664	RT_ZERO(ExitEventInfo);
17665	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17666	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17667	if (pVCpu->hm.s.Event.fPending)
17668	{
17669	Assert(ExitEventInfo.uIdtVectoringInfo == pVCpu->hm.s.Event.u64IntInfo);
17670	Assert(ExitEventInfo.uIdtVectoringErrCode == pVCpu->hm.s.Event.u32ErrCode);
17671	pVCpu->hm.s.Event.fPending = false;
17672	}
17673	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17674	}
17675
17676
17677	/**
17678	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17679	* Conditional VM-exit.
17680	*/
17681	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17682	{
17683	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17684
17685	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17686	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17687	AssertRCReturn(rc, rc);
17688
17689	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17690	}
17691
17692
17693	/**
17694	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17695	* Conditional VM-exit.
17696	*/
17697	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17698	{
17699	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17700
17701	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17702	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17703	AssertRCReturn(rc, rc);
17704
17705	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17706	}
17707
17708
17709	/**
17710	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17711	*/
17712	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17713	{
17714	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17715
17716	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17717	{
17718	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17719	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17720	AssertRCReturn(rc, rc);
17721	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17722	}
17723	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17724	}
17725
17726
17727	/**
17728	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17729	*/
17730	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17731	{
17732	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17733
17734	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17735	{
17736	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17737	AssertRCReturn(rc, rc);
17738	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17739	}
17740	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17741	}
17742
17743
17744	/**
17745	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17746	*/
17747	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17748	{
17749	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17750
17751	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17752	{
17753	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17754	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17755	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17756	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17757	AssertRCReturn(rc, rc);
17758
17759	VMXVEXITINFO ExitInfo;
17760	RT_ZERO(ExitInfo);
17761	ExitInfo.uReason = pVmxTransient->uExitReason;
17762	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17763	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17764	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17765	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17766	}
17767	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17768	}
17769
17770
17771	/**
17772	* Nested-guest VM-exit handler for invalid-guest state
17773	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17774	*/
17775	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17776	{
17777	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17778
17779	/*
17780	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17781	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17782	* Handle it like it's in an invalid guest state of the outer guest.
17783	*
17784	* When the fast path is implemented, this should be changed to cause the corresponding
17785	* nested-guest VM-exit.
17786	*/
17787	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17788	}
17789
17790
17791	/**
17792	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17793	* and only provide the instruction length.
17794	*
17795	* Unconditional VM-exit.
17796	*/
17797	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17798	{
17799	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17800
17801	#ifdef VBOX_STRICT
17802	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17803	switch (pVmxTransient->uExitReason)
17804	{
17805	case VMX_EXIT_ENCLS:
17806	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17807	break;
17808
17809	case VMX_EXIT_VMFUNC:
17810	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17811	break;
17812	}
17813	#endif
17814
17815	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17816	AssertRCReturn(rc, rc);
17817	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17818	}
17819
17820
17821	/**
17822	* Nested-guest VM-exit handler for instructions that provide instruction length as
17823	* well as more information.
17824	*
17825	* Unconditional VM-exit.
17826	*/
17827	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17828	{
17829	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17830
17831	#ifdef VBOX_STRICT
17832	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17833	switch (pVmxTransient->uExitReason)
17834	{
17835	case VMX_EXIT_GDTR_IDTR_ACCESS:
17836	case VMX_EXIT_LDTR_TR_ACCESS:
17837	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17838	break;
17839
17840	case VMX_EXIT_RDRAND:
17841	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17842	break;
17843
17844	case VMX_EXIT_RDSEED:
17845	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17846	break;
17847
17848	case VMX_EXIT_XSAVES:
17849	case VMX_EXIT_XRSTORS:
17850	/** @todo NSTVMX: Verify XSS-bitmap. */
17851	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17852	break;
17853
17854	case VMX_EXIT_UMWAIT:
17855	case VMX_EXIT_TPAUSE:
17856	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17857	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17858	break;
17859	}
17860	#endif
17861
17862	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17863	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17864	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17865	AssertRCReturn(rc, rc);
17866
17867	VMXVEXITINFO ExitInfo;
17868	RT_ZERO(ExitInfo);
17869	ExitInfo.uReason = pVmxTransient->uExitReason;
17870	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17871	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17872	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17873	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17874	}
17875
17876	/** @} */
17877	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17878

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79695

Download in other formats: