HMVMXR0.cpp@ 79698

Last change on this file since 79698 was 79698, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 comments.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 737.4 KB

Line
1	/* $Id: HMVMXR0.cpp 79698 2019-07-11 13:54:20Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/gim.h>
38	#include <VBox/vmm/apic.h>
39	#ifdef VBOX_WITH_REM
40	# include <VBox/vmm/rem.h>
41	#endif
42	#include "HMInternal.h"
43	#include <VBox/vmm/vm.h>
44	#include <VBox/vmm/hmvmxinline.h>
45	#include "HMVMXR0.h"
46	#include "dtrace/VBoxVMM.h"
47
48	#ifdef DEBUG_ramshankar
49	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
50	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
53	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
54	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
55	# define HMVMX_ALWAYS_TRAP_PF
56	# define HMVMX_ALWAYS_FLUSH_TLB
57	# define HMVMX_ALWAYS_SWAP_EFER
58	#endif
59
60
61	/*********************************************************************************************************************************
62	* Defined Constants And Macros *
63	*********************************************************************************************************************************/
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/** @name HMVMX_READ_XXX
74	* Flags to skip redundant reads of some common VMCS fields that are not part of
75	* the guest-CPU or VCPU state but are needed while handling VM-exits.
76	*/
77	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
78	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
79	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
80	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
81	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
82	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
83	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
84	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
85	/** @} */
86
87	/**
88	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
89	* guest using hardware-assisted VMX.
90	*
91	* This excludes state like GPRs (other than RSP) which are always are
92	* swapped and restored across the world-switch and also registers like EFER,
93	* MSR which cannot be modified by the guest without causing a VM-exit.
94	*/
95	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
96	\| CPUMCTX_EXTRN_RFLAGS \
97	\| CPUMCTX_EXTRN_RSP \
98	\| CPUMCTX_EXTRN_SREG_MASK \
99	\| CPUMCTX_EXTRN_TABLE_MASK \
100	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
101	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
102	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
103	\| CPUMCTX_EXTRN_TSC_AUX \
104	\| CPUMCTX_EXTRN_OTHER_MSRS \
105	\| CPUMCTX_EXTRN_CR0 \
106	\| CPUMCTX_EXTRN_CR3 \
107	\| CPUMCTX_EXTRN_CR4 \
108	\| CPUMCTX_EXTRN_DR7 \
109	\| CPUMCTX_EXTRN_HWVIRT \
110	\| CPUMCTX_EXTRN_HM_VMX_MASK)
111
112	/**
113	* Exception bitmap mask for real-mode guests (real-on-v86).
114	*
115	* We need to intercept all exceptions manually except:
116	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
117	* due to bugs in Intel CPUs.
118	* - \#PF need not be intercepted even in real-mode if we have nested paging
119	* support.
120	*/
121	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
122	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
123	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
124	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
125	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
126	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
127	\| RT_BIT(X86_XCPT_XF))
128
129	/** Maximum VM-instruction error number. */
130	#define HMVMX_INSTR_ERROR_MAX 28
131
132	/** Profiling macro. */
133	#ifdef HM_PROFILE_EXIT_DISPATCH
134	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
135	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
136	#else
137	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
138	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
139	#endif
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
151
152	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
153	* context. */
154	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
155	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
156	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
157
158	/** Log the VM-exit reason with an easily visible marker to identify it in a
159	* potential sea of logging data. */
160	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
161	do { \
162	Log4(("VM-exit: vcpu[%RU32] reason=%#x -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", \
163	(a_pVCpu)->idCpu, (a_uExitReason))); \
164	} while (0) \
165
166
167	/*********************************************************************************************************************************
168	* Structures and Typedefs *
169	*********************************************************************************************************************************/
170	/**
171	* VMX per-VCPU transient state.
172	*
173	* A state structure for holding miscellaneous information across
174	* VMX non-root operation and restored after the transition.
175	*/
176	typedef struct VMXTRANSIENT
177	{
178	/** The host's rflags/eflags. */
179	RTCCUINTREG fEFlags;
180	#if HC_ARCH_BITS == 32
181	uint32_t u32Alignment0;
182	#endif
183	/** The guest's TPR value used for TPR shadowing. */
184	uint8_t u8GuestTpr;
185	/** Alignment. */
186	uint8_t abAlignment0[7];
187
188	/** The basic VM-exit reason. */
189	uint16_t uExitReason;
190	/** Alignment. */
191	uint16_t u16Alignment0;
192	/** The VM-exit interruption error code. */
193	uint32_t uExitIntErrorCode;
194	/** The VM-exit exit code qualification. */
195	uint64_t uExitQual;
196	/** The Guest-linear address. */
197	uint64_t uGuestLinearAddr;
198
199	/** The VM-exit interruption-information field. */
200	uint32_t uExitIntInfo;
201	/** The VM-exit instruction-length field. */
202	uint32_t cbInstr;
203	/** The VM-exit instruction-information field. */
204	VMXEXITINSTRINFO ExitInstrInfo;
205	/** Whether the VM-entry failed or not. */
206	bool fVMEntryFailed;
207	/** Whether we are currently executing a nested-guest. */
208	bool fIsNestedGuest;
209	/** Alignment. */
210	uint8_t abAlignment1[2];
211
212	/** The VM-entry interruption-information field. */
213	uint32_t uEntryIntInfo;
214	/** The VM-entry exception error code field. */
215	uint32_t uEntryXcptErrorCode;
216	/** The VM-entry instruction length field. */
217	uint32_t cbEntryInstr;
218
219	/** IDT-vectoring information field. */
220	uint32_t uIdtVectoringInfo;
221	/** IDT-vectoring error code. */
222	uint32_t uIdtVectoringErrorCode;
223
224	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
225	uint32_t fVmcsFieldsRead;
226
227	/** Whether the guest debug state was active at the time of VM-exit. */
228	bool fWasGuestDebugStateActive;
229	/** Whether the hyper debug state was active at the time of VM-exit. */
230	bool fWasHyperDebugStateActive;
231	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
232	bool fUpdatedTscOffsettingAndPreemptTimer;
233	/** Whether the VM-exit was caused by a page-fault during delivery of a
234	* contributory exception or a page-fault. */
235	bool fVectoringDoublePF;
236	/** Whether the VM-exit was caused by a page-fault during delivery of an
237	* external interrupt or NMI. */
238	bool fVectoringPF;
239	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
240	* area after VM-exit. */
241	bool fRemoveTscAuxMsr;
242	bool afAlignment0[2];
243
244	/** The VMCS info. object. */
245	PVMXVMCSINFO pVmcsInfo;
246	} VMXTRANSIENT;
247	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
248	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
249	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
250	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
251	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
252	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
253	/** Pointer to VMX transient state. */
254	typedef VMXTRANSIENT *PVMXTRANSIENT;
255	/** Pointer to a const VMX transient state. */
256	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
257
258	/**
259	* Memory operand read or write access.
260	*/
261	typedef enum VMXMEMACCESS
262	{
263	VMXMEMACCESS_READ = 0,
264	VMXMEMACCESS_WRITE = 1
265	} VMXMEMACCESS;
266
267	/**
268	* VMX VM-exit handler.
269	*
270	* @returns Strict VBox status code (i.e. informational status codes too).
271	* @param pVCpu The cross context virtual CPU structure.
272	* @param pVmxTransient The VMX-transient structure.
273	*/
274	#ifndef HMVMX_USE_FUNCTION_TABLE
275	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
276	#else
277	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
278	/** Pointer to VM-exit handler. */
279	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
280	#endif
281
282	/**
283	* VMX VM-exit handler, non-strict status code.
284	*
285	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
286	*
287	* @returns VBox status code, no informational status code returned.
288	* @param pVCpu The cross context virtual CPU structure.
289	* @param pVmxTransient The VMX-transient structure.
290	*
291	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
292	* use of that status code will be replaced with VINF_EM_SOMETHING
293	* later when switching over to IEM.
294	*/
295	#ifndef HMVMX_USE_FUNCTION_TABLE
296	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
297	#else
298	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
299	#endif
300
301
302	/*********************************************************************************************************************************
303	* Internal Functions *
304	*********************************************************************************************************************************/
305	#ifndef HMVMX_USE_FUNCTION_TABLE
306	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
307	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
308	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
309	#else
310	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
311	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
312	#endif
313	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
314	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
315	#endif
316
317	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
318	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
319	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
320	#endif
321
322	/** @name VM-exit handler prototypes.
323	* @{
324	*/
325	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
326	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
327	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
328	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
329	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
330	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
331	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
332	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
333	static FNVMXEXITHANDLER hmR0VmxExitHlt;
334	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
335	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
336	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
337	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
338	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
339	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
340	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
341	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
342	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
343	static FNVMXEXITHANDLER hmR0VmxExitVmread;
344	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
345	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
346	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
347	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
348	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
349	#endif
350	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
351	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
352	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
353	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
354	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
355	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
356	static FNVMXEXITHANDLER hmR0VmxExitMwait;
357	static FNVMXEXITHANDLER hmR0VmxExitMtf;
358	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
359	static FNVMXEXITHANDLER hmR0VmxExitPause;
360	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
361	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
362	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
363	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
364	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
365	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
367	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
368	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
370	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
371	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
372	/** @} */
373
374	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
375	/** @name Nested-guest VM-exit handler prototypes.
376	* @{
377	*/
378	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
379	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
380	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
381	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
382	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
383	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
384	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
385	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
386	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
387	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
388	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
389	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
390	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
391	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
392	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
393	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
394	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
395	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
396	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
398	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
399	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
400	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
405	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
406	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
407	/** @} */
408	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
409
410
411	/*********************************************************************************************************************************
412	* Global Variables *
413	*********************************************************************************************************************************/
414	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
415	/**
416	* Array of all VMCS fields.
417	* Any fields added to the VT-x spec. should be added here.
418	*
419	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
420	* of nested-guests.
421	*/
422	static const uint32_t g_aVmcsFields[] =
423	{
424	/* 16-bit control fields. */
425	VMX_VMCS16_VPID,
426	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
427	VMX_VMCS16_EPTP_INDEX,
428
429	/* 16-bit guest-state fields. */
430	VMX_VMCS16_GUEST_ES_SEL,
431	VMX_VMCS16_GUEST_CS_SEL,
432	VMX_VMCS16_GUEST_SS_SEL,
433	VMX_VMCS16_GUEST_DS_SEL,
434	VMX_VMCS16_GUEST_FS_SEL,
435	VMX_VMCS16_GUEST_GS_SEL,
436	VMX_VMCS16_GUEST_LDTR_SEL,
437	VMX_VMCS16_GUEST_TR_SEL,
438	VMX_VMCS16_GUEST_INTR_STATUS,
439	VMX_VMCS16_GUEST_PML_INDEX,
440
441	/* 16-bits host-state fields. */
442	VMX_VMCS16_HOST_ES_SEL,
443	VMX_VMCS16_HOST_CS_SEL,
444	VMX_VMCS16_HOST_SS_SEL,
445	VMX_VMCS16_HOST_DS_SEL,
446	VMX_VMCS16_HOST_FS_SEL,
447	VMX_VMCS16_HOST_GS_SEL,
448	VMX_VMCS16_HOST_TR_SEL,
449
450	/* 64-bit control fields. */
451	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
452	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
453	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
454	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
455	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
456	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
457	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
458	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
459	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
460	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
461	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
462	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
463	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
464	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
465	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
466	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
467	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
468	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
469	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
470	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
471	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
472	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
473	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
474	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
475	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
476	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
477	VMX_VMCS64_CTRL_EPTP_FULL,
478	VMX_VMCS64_CTRL_EPTP_HIGH,
479	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
480	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
481	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
482	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
483	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
484	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
485	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
486	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
487	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
488	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
489	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
490	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
491	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
492	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
493	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
494	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
495	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
496	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
497	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
498	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
499	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
500	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
501
502	/* 64-bit read-only data fields. */
503	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
504	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
505
506	/* 64-bit guest-state fields. */
507	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
508	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
509	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
510	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
511	VMX_VMCS64_GUEST_PAT_FULL,
512	VMX_VMCS64_GUEST_PAT_HIGH,
513	VMX_VMCS64_GUEST_EFER_FULL,
514	VMX_VMCS64_GUEST_EFER_HIGH,
515	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
516	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
517	VMX_VMCS64_GUEST_PDPTE0_FULL,
518	VMX_VMCS64_GUEST_PDPTE0_HIGH,
519	VMX_VMCS64_GUEST_PDPTE1_FULL,
520	VMX_VMCS64_GUEST_PDPTE1_HIGH,
521	VMX_VMCS64_GUEST_PDPTE2_FULL,
522	VMX_VMCS64_GUEST_PDPTE2_HIGH,
523	VMX_VMCS64_GUEST_PDPTE3_FULL,
524	VMX_VMCS64_GUEST_PDPTE3_HIGH,
525	VMX_VMCS64_GUEST_BNDCFGS_FULL,
526	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
527
528	/* 64-bit host-state fields. */
529	VMX_VMCS64_HOST_PAT_FULL,
530	VMX_VMCS64_HOST_PAT_HIGH,
531	VMX_VMCS64_HOST_EFER_FULL,
532	VMX_VMCS64_HOST_EFER_HIGH,
533	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
534	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
535
536	/* 32-bit control fields. */
537	VMX_VMCS32_CTRL_PIN_EXEC,
538	VMX_VMCS32_CTRL_PROC_EXEC,
539	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
540	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
541	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
542	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
543	VMX_VMCS32_CTRL_EXIT,
544	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
545	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
546	VMX_VMCS32_CTRL_ENTRY,
547	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
548	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
549	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
550	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
551	VMX_VMCS32_CTRL_TPR_THRESHOLD,
552	VMX_VMCS32_CTRL_PROC_EXEC2,
553	VMX_VMCS32_CTRL_PLE_GAP,
554	VMX_VMCS32_CTRL_PLE_WINDOW,
555
556	/* 32-bits read-only fields. */
557	VMX_VMCS32_RO_VM_INSTR_ERROR,
558	VMX_VMCS32_RO_EXIT_REASON,
559	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
560	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
561	VMX_VMCS32_RO_IDT_VECTORING_INFO,
562	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
563	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
564	VMX_VMCS32_RO_EXIT_INSTR_INFO,
565
566	/* 32-bit guest-state fields. */
567	VMX_VMCS32_GUEST_ES_LIMIT,
568	VMX_VMCS32_GUEST_CS_LIMIT,
569	VMX_VMCS32_GUEST_SS_LIMIT,
570	VMX_VMCS32_GUEST_DS_LIMIT,
571	VMX_VMCS32_GUEST_FS_LIMIT,
572	VMX_VMCS32_GUEST_GS_LIMIT,
573	VMX_VMCS32_GUEST_LDTR_LIMIT,
574	VMX_VMCS32_GUEST_TR_LIMIT,
575	VMX_VMCS32_GUEST_GDTR_LIMIT,
576	VMX_VMCS32_GUEST_IDTR_LIMIT,
577	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
578	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
579	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
580	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
581	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
582	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
583	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
584	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
585	VMX_VMCS32_GUEST_INT_STATE,
586	VMX_VMCS32_GUEST_ACTIVITY_STATE,
587	VMX_VMCS32_GUEST_SMBASE,
588	VMX_VMCS32_GUEST_SYSENTER_CS,
589	VMX_VMCS32_PREEMPT_TIMER_VALUE,
590
591	/* 32-bit host-state fields. */
592	VMX_VMCS32_HOST_SYSENTER_CS,
593
594	/* Natural-width control fields. */
595	VMX_VMCS_CTRL_CR0_MASK,
596	VMX_VMCS_CTRL_CR4_MASK,
597	VMX_VMCS_CTRL_CR0_READ_SHADOW,
598	VMX_VMCS_CTRL_CR4_READ_SHADOW,
599	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
600	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
601	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
602	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
603
604	/* Natural-width read-only data fields. */
605	VMX_VMCS_RO_EXIT_QUALIFICATION,
606	VMX_VMCS_RO_IO_RCX,
607	VMX_VMCS_RO_IO_RSI,
608	VMX_VMCS_RO_IO_RDI,
609	VMX_VMCS_RO_IO_RIP,
610	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
611
612	/* Natural-width guest-state field */
613	VMX_VMCS_GUEST_CR0,
614	VMX_VMCS_GUEST_CR3,
615	VMX_VMCS_GUEST_CR4,
616	VMX_VMCS_GUEST_ES_BASE,
617	VMX_VMCS_GUEST_CS_BASE,
618	VMX_VMCS_GUEST_SS_BASE,
619	VMX_VMCS_GUEST_DS_BASE,
620	VMX_VMCS_GUEST_FS_BASE,
621	VMX_VMCS_GUEST_GS_BASE,
622	VMX_VMCS_GUEST_LDTR_BASE,
623	VMX_VMCS_GUEST_TR_BASE,
624	VMX_VMCS_GUEST_GDTR_BASE,
625	VMX_VMCS_GUEST_IDTR_BASE,
626	VMX_VMCS_GUEST_DR7,
627	VMX_VMCS_GUEST_RSP,
628	VMX_VMCS_GUEST_RIP,
629	VMX_VMCS_GUEST_RFLAGS,
630	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
631	VMX_VMCS_GUEST_SYSENTER_ESP,
632	VMX_VMCS_GUEST_SYSENTER_EIP,
633
634	/* Natural-width host-state fields */
635	VMX_VMCS_HOST_CR0,
636	VMX_VMCS_HOST_CR3,
637	VMX_VMCS_HOST_CR4,
638	VMX_VMCS_HOST_FS_BASE,
639	VMX_VMCS_HOST_GS_BASE,
640	VMX_VMCS_HOST_TR_BASE,
641	VMX_VMCS_HOST_GDTR_BASE,
642	VMX_VMCS_HOST_IDTR_BASE,
643	VMX_VMCS_HOST_SYSENTER_ESP,
644	VMX_VMCS_HOST_SYSENTER_EIP,
645	VMX_VMCS_HOST_RSP,
646	VMX_VMCS_HOST_RIP
647	};
648	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
649
650	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
651	static const uint32_t g_aVmcsCacheSegBase[] =
652	{
653	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
654	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
655	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
656	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
657	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
658	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
659	};
660	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
661	#endif
662	static const uint32_t g_aVmcsSegBase[] =
663	{
664	VMX_VMCS_GUEST_ES_BASE,
665	VMX_VMCS_GUEST_CS_BASE,
666	VMX_VMCS_GUEST_SS_BASE,
667	VMX_VMCS_GUEST_DS_BASE,
668	VMX_VMCS_GUEST_FS_BASE,
669	VMX_VMCS_GUEST_GS_BASE
670	};
671	static const uint32_t g_aVmcsSegSel[] =
672	{
673	VMX_VMCS16_GUEST_ES_SEL,
674	VMX_VMCS16_GUEST_CS_SEL,
675	VMX_VMCS16_GUEST_SS_SEL,
676	VMX_VMCS16_GUEST_DS_SEL,
677	VMX_VMCS16_GUEST_FS_SEL,
678	VMX_VMCS16_GUEST_GS_SEL
679	};
680	static const uint32_t g_aVmcsSegLimit[] =
681	{
682	VMX_VMCS32_GUEST_ES_LIMIT,
683	VMX_VMCS32_GUEST_CS_LIMIT,
684	VMX_VMCS32_GUEST_SS_LIMIT,
685	VMX_VMCS32_GUEST_DS_LIMIT,
686	VMX_VMCS32_GUEST_FS_LIMIT,
687	VMX_VMCS32_GUEST_GS_LIMIT
688	};
689	static const uint32_t g_aVmcsSegAttr[] =
690	{
691	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
692	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
693	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
694	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
695	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
696	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
697	};
698	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
699	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
700	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
701	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
702
703	#ifdef HMVMX_USE_FUNCTION_TABLE
704	/**
705	* VMX_EXIT dispatch table.
706	*/
707	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
708	{
709	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
710	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
711	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
712	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
713	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
714	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
715	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
716	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
717	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
718	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
719	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
720	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
721	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
722	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
723	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
724	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
725	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
726	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
727	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
728	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
729	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
730	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
731	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
732	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
733	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
734	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
735	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
736	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
737	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
738	#else
739	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
740	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
741	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
742	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
743	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
744	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
745	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
746	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
747	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
748	#endif
749	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
750	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
751	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
752	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
753	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
754	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
755	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
756	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
757	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
758	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
759	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
760	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
761	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
762	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
763	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
765	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
766	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
767	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
768	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
769	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
770	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
771	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
772	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
773	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
774	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
775	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
776	#else
777	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
778	#endif
779	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
780	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
781	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
782	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
783	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
784	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
785	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
786	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
787	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
788	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
789	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
790	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
791	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
792	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
793	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
794	};
795	#endif /* HMVMX_USE_FUNCTION_TABLE */
796
797	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
798	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
799	{
800	/* 0 */ "(Not Used)",
801	/* 1 */ "VMCALL executed in VMX root operation.",
802	/* 2 */ "VMCLEAR with invalid physical address.",
803	/* 3 */ "VMCLEAR with VMXON pointer.",
804	/* 4 */ "VMLAUNCH with non-clear VMCS.",
805	/* 5 */ "VMRESUME with non-launched VMCS.",
806	/* 6 */ "VMRESUME after VMXOFF",
807	/* 7 */ "VM-entry with invalid control fields.",
808	/* 8 */ "VM-entry with invalid host state fields.",
809	/* 9 */ "VMPTRLD with invalid physical address.",
810	/* 10 */ "VMPTRLD with VMXON pointer.",
811	/* 11 */ "VMPTRLD with incorrect revision identifier.",
812	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
813	/* 13 */ "VMWRITE to read-only VMCS component.",
814	/* 14 */ "(Not Used)",
815	/* 15 */ "VMXON executed in VMX root operation.",
816	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
817	/* 17 */ "VM-entry with non-launched executing VMCS.",
818	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
819	/* 19 */ "VMCALL with non-clear VMCS.",
820	/* 20 */ "VMCALL with invalid VM-exit control fields.",
821	/* 21 */ "(Not Used)",
822	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
823	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
824	/* 24 */ "VMCALL with invalid SMM-monitor features.",
825	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
826	/* 26 */ "VM-entry with events blocked by MOV SS.",
827	/* 27 */ "(Not Used)",
828	/* 28 */ "Invalid operand to INVEPT/INVVPID."
829	};
830	#endif /* VBOX_STRICT && LOG_ENABLED */
831
832
833	/**
834	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
835	*
836	* Any bit set in this mask is owned by the host/hypervisor and would cause a
837	* VM-exit when modified by the guest.
838	*
839	* @returns The static CR0 guest/host mask.
840	* @param pVCpu The cross context virtual CPU structure.
841	*/
842	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
843	{
844	/*
845	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
846	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
847	*/
848	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
849	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
850	* and @bugref{6944}. */
851	PVM pVM = pVCpu->CTX_SUFF(pVM);
852	return ( X86_CR0_PE
853	\| X86_CR0_NE
854	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
855	\| X86_CR0_PG
856	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
857	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
858	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
859	}
860
861
862	/**
863	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
864	*
865	* Any bit set in this mask is owned by the host/hypervisor and would cause a
866	* VM-exit when modified by the guest.
867	*
868	* @returns The static CR4 guest/host mask.
869	* @param pVCpu The cross context virtual CPU structure.
870	*/
871	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
872	{
873	/*
874	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
875	* these bits are reserved on hardware that does not support them. Since the
876	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
877	* these bits and handle it depending on whether we expose them to the guest.
878	*/
879	PVM pVM = pVCpu->CTX_SUFF(pVM);
880	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
881	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
882	return ( X86_CR4_VMXE
883	\| X86_CR4_VME
884	\| X86_CR4_PAE
885	\| X86_CR4_PGE
886	\| X86_CR4_PSE
887	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
888	\| (fPcid ? X86_CR4_PCIDE : 0));
889	}
890
891
892	/**
893	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
894	* area.
895	*
896	* @returns @c true if it's different, @c false otherwise.
897	* @param pVmcsInfo The VMCS info. object.
898	*/
899	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
900	{
901	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
902	&& pVmcsInfo->pvGuestMsrStore);
903	}
904
905
906	/**
907	* Checks whether one of the given Pin-based VM-execution controls are set.
908	*
909	* @returns @c true if set, @c false otherwise.
910	* @param pVCpu The cross context virtual CPU structure.
911	* @param pVmxTransient The VMX-transient structure.
912	* @param uPinCtls The Pin-based VM-execution controls to check.
913	*
914	* @remarks This will not check merged controls when executing a nested-guest
915	* but the original control specified by the guest hypervisor.
916	*/
917	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
918	{
919	if (!pVmxTransient->fIsNestedGuest)
920	{
921	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
922	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
923	}
924	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
925	}
926
927
928	/**
929	* Sets the given Processor-based VM-execution controls.
930	*
931	* @param pVmxTransient The VMX-transient structure.
932	* @param uProcCtls The Processor-based VM-execution controls to set.
933	*/
934	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
935	{
936	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
937	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
938	{
939	pVmcsInfo->u32ProcCtls \|= uProcCtls;
940	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
941	AssertRC(rc);
942	}
943	}
944
945
946	/**
947	* Removes the given Processor-based VM-execution controls.
948	*
949	* @param pVCpu The cross context virtual CPU structure.
950	* @param pVmxTransient The VMX-transient structure.
951	* @param uProcCtls The Processor-based VM-execution controls to remove.
952	*
953	* @remarks When executing a nested-guest, this will not remove any of the specified
954	* controls if the guest hypervisor has set any one of them.
955	*/
956	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
957	{
958	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
959	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
960	? true
961	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT);
962	#else
963	NOREF(pVCpu);
964	bool const fRemoveCtls = true;
965	#endif
966	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
967	if ( fRemoveCtls
968	&& (pVmcsInfo->u32ProcCtls & uProcCtls))
969	{
970	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
971	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
972	AssertRC(rc);
973	}
974	}
975
976
977	/**
978	* Sets the TSC offset for the current VMCS.
979	*
980	* @param pVCpu The cross context virtual CPU structure.
981	* @param uTscOffset The TSC offset to set.
982	* @param pVmcsInfo The VMCS info. object.
983	*/
984	static void hmR0VmxSetTscOffsetVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
985	{
986	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
987	if (pVmcsInfo->u64TscOffset != uTscOffset)
988	{
989	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
990	AssertRC(rc);
991	pVmcsInfo->u64TscOffset = uTscOffset;
992	}
993	}
994
995
996	/**
997	* Adds one or more exceptions to the exception bitmap and commits it to the current
998	* VMCS.
999	*
1000	* @returns VBox status code.
1001	* @param pVmxTransient The VMX-transient structure.
1002	* @param uXcptMask The exception(s) to add.
1003	*/
1004	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1005	{
1006	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1007	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1008	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1009	{
1010	uXcptBitmap \|= uXcptMask;
1011	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1012	AssertRCReturn(rc, rc);
1013	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1014	}
1015	return VINF_SUCCESS;
1016	}
1017
1018
1019	/**
1020	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1021	*
1022	* @returns VBox status code.
1023	* @param pVmxTransient The VMX-transient structure.
1024	* @param uXcpt The exception to add.
1025	*/
1026	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1027	{
1028	Assert(uXcpt <= X86_XCPT_LAST);
1029	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1030	}
1031
1032
1033	/**
1034	* Remove one or more exceptions from the exception bitmap and commits it to the
1035	* current VMCS.
1036	*
1037	* This takes care of not removing the exception intercept if a nested-guest
1038	* requires the exception to be intercepted.
1039	*
1040	* @returns VBox status code.
1041	* @param pVCpu The cross context virtual CPU structure.
1042	* @param pVmxTransient The VMX-transient structure.
1043	* @param uXcptMask The exception(s) to remove.
1044	*/
1045	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1046	{
1047	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1048	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1049	if (u32XcptBitmap & uXcptMask)
1050	{
1051	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1052	if (!pVmxTransient->fIsNestedGuest)
1053	{ /* likely */ }
1054	else
1055	{
1056	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1057	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1058	}
1059	#endif
1060	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1061	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1062	\| RT_BIT(X86_XCPT_DE)
1063	\| RT_BIT(X86_XCPT_NM)
1064	\| RT_BIT(X86_XCPT_TS)
1065	\| RT_BIT(X86_XCPT_UD)
1066	\| RT_BIT(X86_XCPT_NP)
1067	\| RT_BIT(X86_XCPT_SS)
1068	\| RT_BIT(X86_XCPT_GP)
1069	\| RT_BIT(X86_XCPT_PF)
1070	\| RT_BIT(X86_XCPT_MF));
1071	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1072	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1073	#endif
1074	if (uXcptMask)
1075	{
1076	/* Validate we are not removing any essential exception intercepts. */
1077	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1078	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1079	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1080
1081	/* Remove it from the exception bitmap. */
1082	u32XcptBitmap &= ~uXcptMask;
1083
1084	/* Commit and update the cache if necessary. */
1085	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1086	{
1087	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1088	AssertRCReturn(rc, rc);
1089	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1090	}
1091	}
1092	}
1093	return VINF_SUCCESS;
1094	}
1095
1096
1097	/**
1098	* Remove an exceptions from the exception bitmap and commits it to the current
1099	* VMCS.
1100	*
1101	* @returns VBox status code.
1102	* @param pVCpu The cross context virtual CPU structure.
1103	* @param pVmxTransient The VMX-transient structure.
1104	* @param uXcpt The exception to remove.
1105	*/
1106	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1107	{
1108	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1109	}
1110
1111
1112	/**
1113	* Loads the VMCS specified by the VMCS info. object.
1114	*
1115	* @returns VBox status code.
1116	* @param pVmcsInfo The VMCS info. object.
1117	*
1118	* @remarks Can be called with interrupts disabled.
1119	*/
1120	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1121	{
1122	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1123	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1124
1125	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1126	if (RT_SUCCESS(rc))
1127	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1128	return rc;
1129	}
1130
1131
1132	/**
1133	* Clears the VMCS specified by the VMCS info. object.
1134	*
1135	* @returns VBox status code.
1136	* @param pVmcsInfo The VMCS info. object.
1137	*
1138	* @remarks Can be called with interrupts disabled.
1139	*/
1140	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1141	{
1142	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1143	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1144
1145	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1146	if (RT_SUCCESS(rc))
1147	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1148	return rc;
1149	}
1150
1151
1152	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1153	/**
1154	* Loads the shadow VMCS specified by the VMCS info. object.
1155	*
1156	* @returns VBox status code.
1157	* @param pVmcsInfo The VMCS info. object.
1158	*
1159	* @remarks Can be called with interrupts disabled.
1160	*/
1161	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1162	{
1163	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1164	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1165
1166	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1167	if (RT_SUCCESS(rc))
1168	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1169	return rc;
1170	}
1171
1172
1173	/**
1174	* Clears the shadow VMCS specified by the VMCS info. object.
1175	*
1176	* @returns VBox status code.
1177	* @param pVmcsInfo The VMCS info. object.
1178	*
1179	* @remarks Can be called with interrupts disabled.
1180	*/
1181	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1182	{
1183	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1184	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1185
1186	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1187	if (RT_SUCCESS(rc))
1188	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1189	return rc;
1190	}
1191
1192
1193	/**
1194	* Switches from and to the specified VMCSes.
1195	*
1196	* @returns VBox status code.
1197	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1198	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1199	*
1200	* @remarks Called with interrupts disabled.
1201	*/
1202	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1203	{
1204	/*
1205	* Clear the VMCS we are switching out if it has not already been cleared.
1206	* This will sync any CPU internal data back to the VMCS.
1207	*/
1208	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1209	{
1210	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1211	if (RT_SUCCESS(rc))
1212	{
1213	/*
1214	* The shadow VMCS, if any, would not be active at this point since we
1215	* would have cleared it while importing the virtual hardware-virtualization
1216	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1217	* clear the shadow VMCS here, just assert for safety.
1218	*/
1219	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1220	}
1221	else
1222	return rc;
1223	}
1224
1225	/*
1226	* Clear the VMCS we are switching to if it has not already been cleared.
1227	* This will initialize the VMCS launch state to "clear" required for loading it.
1228	*
1229	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1230	*/
1231	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1232	{
1233	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1234	if (RT_SUCCESS(rc))
1235	{ /* likely */ }
1236	else
1237	return rc;
1238	}
1239
1240	/*
1241	* Finally, load the VMCS we are switching to.
1242	*/
1243	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1244	}
1245
1246
1247	/**
1248	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1249	* caller.
1250	*
1251	* @returns VBox status code.
1252	* @param pVCpu The cross context virtual CPU structure.
1253	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1254	* true) or guest VMCS (pass false).
1255	*/
1256	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1257	{
1258	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1259	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1260
1261	PVMXVMCSINFO pVmcsInfoFrom;
1262	PVMXVMCSINFO pVmcsInfoTo;
1263	if (fSwitchToNstGstVmcs)
1264	{
1265	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1266	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1267	}
1268	else
1269	{
1270	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1271	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1272	}
1273
1274	/*
1275	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1276	* preemption hook code path acquires the current VMCS.
1277	*/
1278	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1279
1280	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1281	if (RT_SUCCESS(rc))
1282	{
1283	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1284
1285	/*
1286	* If we are switching to a VMCS that was executed on a different host CPU or was
1287	* never executed before, flag that we need to export the host state before executing
1288	* guest/nested-guest code using hardware-assisted VMX.
1289	*
1290	* This could probably be done in a preemptible context since the preemption hook
1291	* will flag the necessary change in host context. However, since preemption is
1292	* already disabled and to avoid making assumptions about host specific code in
1293	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1294	* disabled.
1295	*/
1296	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1297	{ /* likely */ }
1298	else
1299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1300
1301	ASMSetFlags(fEFlags);
1302
1303	/*
1304	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1305	* flag that we need to update the host MSR values there. Even if we decide in the
1306	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1307	* if its content differs, we would have to update the host MSRs anyway.
1308	*/
1309	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1310	}
1311	else
1312	ASMSetFlags(fEFlags);
1313	return rc;
1314	}
1315	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1316
1317
1318	/**
1319	* Updates the VM's last error record.
1320	*
1321	* If there was a VMX instruction error, reads the error data from the VMCS and
1322	* updates VCPU's last error record as well.
1323	*
1324	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1325	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1326	* VERR_VMX_INVALID_VMCS_FIELD.
1327	* @param rc The error code.
1328	*/
1329	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1330	{
1331	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1332	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1333	{
1334	AssertPtrReturnVoid(pVCpu);
1335	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1336	}
1337	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1338	}
1339
1340
1341	#ifdef VBOX_STRICT
1342	/**
1343	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1344	* transient structure.
1345	*
1346	* @returns VBox status code.
1347	* @param pVmxTransient The VMX-transient structure.
1348	*
1349	* @remarks No-long-jump zone!!!
1350	*/
1351	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1352	{
1353	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1354	AssertRCReturn(rc, rc);
1355	return VINF_SUCCESS;
1356	}
1357
1358
1359	/**
1360	* Reads the VM-entry exception error code field from the VMCS into
1361	* the VMX transient structure.
1362	*
1363	* @returns VBox status code.
1364	* @param pVmxTransient The VMX-transient structure.
1365	*
1366	* @remarks No-long-jump zone!!!
1367	*/
1368	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1369	{
1370	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1371	AssertRCReturn(rc, rc);
1372	return VINF_SUCCESS;
1373	}
1374
1375
1376	/**
1377	* Reads the VM-entry exception error code field from the VMCS into
1378	* the VMX transient structure.
1379	*
1380	* @returns VBox status code.
1381	* @param pVmxTransient The VMX-transient structure.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1386	{
1387	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1388	AssertRCReturn(rc, rc);
1389	return VINF_SUCCESS;
1390	}
1391	#endif /* VBOX_STRICT */
1392
1393
1394	/**
1395	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1396	* transient structure.
1397	*
1398	* @returns VBox status code.
1399	* @param pVmxTransient The VMX-transient structure.
1400	*/
1401	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1402	{
1403	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1404	{
1405	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1406	AssertRCReturn(rc,rc);
1407	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1408	}
1409	return VINF_SUCCESS;
1410	}
1411
1412
1413	/**
1414	* Reads the VM-exit interruption error code from the VMCS into the VMX
1415	* transient structure.
1416	*
1417	* @returns VBox status code.
1418	* @param pVmxTransient The VMX-transient structure.
1419	*/
1420	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1421	{
1422	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1423	{
1424	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1425	AssertRCReturn(rc, rc);
1426	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1427	}
1428	return VINF_SUCCESS;
1429	}
1430
1431
1432	/**
1433	* Reads the VM-exit instruction length field from the VMCS into the VMX
1434	* transient structure.
1435	*
1436	* @returns VBox status code.
1437	* @param pVmxTransient The VMX-transient structure.
1438	*/
1439	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1440	{
1441	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1442	{
1443	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1444	AssertRCReturn(rc, rc);
1445	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1446	}
1447	return VINF_SUCCESS;
1448	}
1449
1450
1451	/**
1452	* Reads the VM-exit instruction-information field from the VMCS into
1453	* the VMX transient structure.
1454	*
1455	* @returns VBox status code.
1456	* @param pVmxTransient The VMX-transient structure.
1457	*/
1458	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1459	{
1460	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1461	{
1462	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1463	AssertRCReturn(rc, rc);
1464	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1465	}
1466	return VINF_SUCCESS;
1467	}
1468
1469
1470	/**
1471	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1472	*
1473	* @returns VBox status code.
1474	* @param pVCpu The cross context virtual CPU structure of the
1475	* calling EMT. (Required for the VMCS cache case.)
1476	* @param pVmxTransient The VMX-transient structure.
1477	*/
1478	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1479	{
1480	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1481	{
1482	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1483	AssertRCReturn(rc, rc);
1484	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1485	}
1486	return VINF_SUCCESS;
1487	}
1488
1489
1490	/**
1491	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1492	*
1493	* @returns VBox status code.
1494	* @param pVCpu The cross context virtual CPU structure of the
1495	* calling EMT. (Required for the VMCS cache case.)
1496	* @param pVmxTransient The VMX-transient structure.
1497	*/
1498	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1499	{
1500	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1501	{
1502	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1503	AssertRCReturn(rc, rc);
1504	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1505	}
1506	return VINF_SUCCESS;
1507	}
1508
1509
1510	/**
1511	* Reads the IDT-vectoring information field from the VMCS into the VMX
1512	* transient structure.
1513	*
1514	* @returns VBox status code.
1515	* @param pVmxTransient The VMX-transient structure.
1516	*
1517	* @remarks No-long-jump zone!!!
1518	*/
1519	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1520	{
1521	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1522	{
1523	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1524	AssertRCReturn(rc, rc);
1525	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1526	}
1527	return VINF_SUCCESS;
1528	}
1529
1530
1531	/**
1532	* Reads the IDT-vectoring error code from the VMCS into the VMX
1533	* transient structure.
1534	*
1535	* @returns VBox status code.
1536	* @param pVmxTransient The VMX-transient structure.
1537	*/
1538	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1539	{
1540	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1541	{
1542	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1543	AssertRCReturn(rc, rc);
1544	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1545	}
1546	return VINF_SUCCESS;
1547	}
1548
1549
1550	/**
1551	* Enters VMX root mode operation on the current CPU.
1552	*
1553	* @returns VBox status code.
1554	* @param pVM The cross context VM structure. Can be
1555	* NULL, after a resume.
1556	* @param HCPhysCpuPage Physical address of the VMXON region.
1557	* @param pvCpuPage Pointer to the VMXON region.
1558	*/
1559	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1560	{
1561	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1562	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1563	Assert(pvCpuPage);
1564	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1565
1566	if (pVM)
1567	{
1568	/* Write the VMCS revision identifier to the VMXON region. */
1569	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1570	}
1571
1572	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1573	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1574
1575	/* Enable the VMX bit in CR4 if necessary. */
1576	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1577
1578	/* Enter VMX root mode. */
1579	int rc = VMXEnable(HCPhysCpuPage);
1580	if (RT_FAILURE(rc))
1581	{
1582	if (!(uOldCr4 & X86_CR4_VMXE))
1583	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1584
1585	if (pVM)
1586	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1587	}
1588
1589	/* Restore interrupts. */
1590	ASMSetFlags(fEFlags);
1591	return rc;
1592	}
1593
1594
1595	/**
1596	* Exits VMX root mode operation on the current CPU.
1597	*
1598	* @returns VBox status code.
1599	*/
1600	static int hmR0VmxLeaveRootMode(void)
1601	{
1602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603
1604	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1605	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1606
1607	/* If we're for some reason not in VMX root mode, then don't leave it. */
1608	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1609
1610	int rc;
1611	if (uHostCr4 & X86_CR4_VMXE)
1612	{
1613	/* Exit VMX root mode and clear the VMX bit in CR4. */
1614	VMXDisable();
1615	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1616	rc = VINF_SUCCESS;
1617	}
1618	else
1619	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1620
1621	/* Restore interrupts. */
1622	ASMSetFlags(fEFlags);
1623	return rc;
1624	}
1625
1626
1627	/**
1628	* Allocates and maps a physically contiguous page. The allocated page is
1629	* zero'd out (used by various VT-x structures).
1630	*
1631	* @returns IPRT status code.
1632	* @param pMemObj Pointer to the ring-0 memory object.
1633	* @param ppVirt Where to store the virtual address of the allocation.
1634	* @param pHCPhys Where to store the physical address of the allocation.
1635	*/
1636	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1637	{
1638	AssertPtr(pMemObj);
1639	AssertPtr(ppVirt);
1640	AssertPtr(pHCPhys);
1641	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1642	if (RT_FAILURE(rc))
1643	return rc;
1644	ppVirt = RTR0MemObjAddress(pMemObj);
1645	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1646	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1647	return VINF_SUCCESS;
1648	}
1649
1650
1651	/**
1652	* Frees and unmaps an allocated, physical page.
1653	*
1654	* @param pMemObj Pointer to the ring-0 memory object.
1655	* @param ppVirt Where to re-initialize the virtual address of allocation as
1656	* 0.
1657	* @param pHCPhys Where to re-initialize the physical address of the
1658	* allocation as 0.
1659	*/
1660	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1661	{
1662	AssertPtr(pMemObj);
1663	AssertPtr(ppVirt);
1664	AssertPtr(pHCPhys);
1665	/* NULL is valid, accepted and ignored by the free function below. */
1666	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1667	*pMemObj = NIL_RTR0MEMOBJ;
1668	*ppVirt = NULL;
1669	*pHCPhys = NIL_RTHCPHYS;
1670	}
1671
1672
1673	/**
1674	* Initializes a VMCS info. object.
1675	*
1676	* @param pVmcsInfo The VMCS info. object.
1677	*/
1678	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1679	{
1680	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1681
1682	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1683	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1684	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1685	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1686	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1687	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1688	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1689	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1690	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1691	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1692	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1693	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1694	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1695	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1696	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1697	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1698	}
1699
1700
1701	/**
1702	* Frees the VT-x structures for a VMCS info. object.
1703	*
1704	* @param pVM The cross context VM structure.
1705	* @param pVmcsInfo The VMCS info. object.
1706	*/
1707	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1708	{
1709	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1710
1711	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1712	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1713	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1714	#endif
1715
1716	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1717	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1718
1719	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1720	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1721	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1722
1723	hmR0VmxInitVmcsInfo(pVmcsInfo);
1724	}
1725
1726
1727	/**
1728	* Allocates the VT-x structures for a VMCS info. object.
1729	*
1730	* @returns VBox status code.
1731	* @param pVCpu The cross context virtual CPU structure.
1732	* @param pVmcsInfo The VMCS info. object.
1733	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1734	*/
1735	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1736	{
1737	PVM pVM = pVCpu->CTX_SUFF(pVM);
1738
1739	/* Allocate the guest VM control structure (VMCS). */
1740	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1741	if (RT_SUCCESS(rc))
1742	{
1743	if (!fIsNstGstVmcs)
1744	{
1745	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1746	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1747	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1748	#endif
1749	if (RT_SUCCESS(rc))
1750	{
1751	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1752	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1753	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1754	{
1755	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1756	NULL /* pR3Ptr /, NULL / pRCPtr */);
1757	}
1758	}
1759	}
1760	else
1761	{
1762	/* We don't yet support exposing VMCS shadowing to the guest. */
1763	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1764	Assert(!pVmcsInfo->pvShadowVmcs);
1765
1766	/* Get the allocated virtual-APIC page from CPUM. */
1767	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1768	{
1769	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1770	&pVmcsInfo->HCPhysVirtApic);
1771	Assert(pVmcsInfo->pbVirtApic);
1772	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1773	}
1774	}
1775
1776	if (RT_SUCCESS(rc))
1777	{
1778	/*
1779	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1780	* transparent accesses of specific MSRs.
1781	*
1782	* If the condition for enabling MSR bitmaps changes here, don't forget to
1783	* update HMIsMsrBitmapActive().
1784	*
1785	* We don't share MSR bitmaps between the guest and nested-guest as we then
1786	* don't need to care about carefully restoring the guest MSR bitmap.
1787	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1788	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1789	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1790	* we do that later while merging VMCS.
1791	*/
1792	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1793	{
1794	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1795	if ( RT_SUCCESS(rc)
1796	&& !fIsNstGstVmcs)
1797	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1798	}
1799
1800	if (RT_SUCCESS(rc))
1801	{
1802	/*
1803	* Allocate the VM-entry MSR-load area for the guest MSRs.
1804	*
1805	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1806	* the guest and nested-guest.
1807	*/
1808	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1809	&pVmcsInfo->HCPhysGuestMsrLoad);
1810	if (RT_SUCCESS(rc))
1811	{
1812	/*
1813	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1814	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1815	*/
1816	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1817	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1818	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1819
1820	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1821	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1822	&pVmcsInfo->HCPhysHostMsrLoad);
1823	}
1824	}
1825	}
1826	}
1827
1828	return rc;
1829	}
1830
1831
1832	/**
1833	* Free all VT-x structures for the VM.
1834	*
1835	* @returns IPRT status code.
1836	* @param pVM The cross context VM structure.
1837	*/
1838	static void hmR0VmxStructsFree(PVM pVM)
1839	{
1840	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1841	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1842	#endif
1843	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1844
1845	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1846	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1847	{
1848	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1849	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1850	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1851	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1852	}
1853	#endif
1854
1855	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1856	{
1857	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1858	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1859	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1860	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1861	if (pVM->cpum.ro.GuestFeatures.fVmx)
1862	{
1863	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1864	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1865	}
1866	#endif
1867	}
1868	}
1869
1870
1871	/**
1872	* Allocate all VT-x structures for the VM.
1873	*
1874	* @returns IPRT status code.
1875	* @param pVM The cross context VM structure.
1876	*/
1877	static int hmR0VmxStructsAlloc(PVM pVM)
1878	{
1879	/*
1880	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1881	* The VMCS size cannot be more than 4096 bytes.
1882	*
1883	* See Intel spec. Appendix A.1 "Basic VMX Information".
1884	*/
1885	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1886	if (cbVmcs <= X86_PAGE_4K_SIZE)
1887	{ /* likely */ }
1888	else
1889	{
1890	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1891	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1892	}
1893
1894	/*
1895	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1896	*/
1897	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1898	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1899	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1900	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1901	#endif
1902
1903	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1904	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1905	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1906
1907	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1908	{
1909	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1910	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1911	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1912	}
1913
1914	/*
1915	* Allocate per-VM VT-x structures.
1916	*/
1917	int rc = VINF_SUCCESS;
1918	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1919	/* Allocate crash-dump magic scratch page. */
1920	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1921	if (RT_FAILURE(rc))
1922	{
1923	hmR0VmxStructsFree(pVM);
1924	return rc;
1925	}
1926	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1927	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1928	#endif
1929
1930	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1931	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1932	{
1933	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1934	&pVM->hm.s.vmx.HCPhysApicAccess);
1935	if (RT_FAILURE(rc))
1936	{
1937	hmR0VmxStructsFree(pVM);
1938	return rc;
1939	}
1940	}
1941
1942	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1943	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1944	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1945	{
1946	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1947	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1948	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1949	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1950	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1951	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1952	{
1953	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1954	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1955	if (RT_SUCCESS(rc))
1956	{
1957	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1958	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1959	}
1960	}
1961	else
1962	rc = VERR_NO_MEMORY;
1963
1964	if (RT_FAILURE(rc))
1965	{
1966	hmR0VmxStructsFree(pVM);
1967	return rc;
1968	}
1969	}
1970	#endif
1971
1972	/*
1973	* Initialize per-VCPU VT-x structures.
1974	*/
1975	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1976	{
1977	/* Allocate the guest VMCS structures. */
1978	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1979	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1980	if (RT_SUCCESS(rc))
1981	{
1982	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1983	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1984	if (pVM->cpum.ro.GuestFeatures.fVmx)
1985	{
1986	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1987	if (RT_SUCCESS(rc))
1988	{ /* likely */ }
1989	else
1990	break;
1991	}
1992	#endif
1993	}
1994	else
1995	break;
1996	}
1997
1998	if (RT_FAILURE(rc))
1999	{
2000	hmR0VmxStructsFree(pVM);
2001	return rc;
2002	}
2003
2004	return VINF_SUCCESS;
2005	}
2006
2007	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2008	/**
2009	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2010	*
2011	* @returns @c true if the MSR is intercepted, @c false otherwise.
2012	* @param pvMsrBitmap The MSR bitmap.
2013	* @param offMsr The MSR byte offset.
2014	* @param iBit The bit offset from the byte offset.
2015	*/
2016	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2017	{
2018	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2019	Assert(pbMsrBitmap);
2020	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2021	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2022	}
2023	#endif
2024
2025	/**
2026	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2027	*
2028	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2029	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2030	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2031	* the read/write access of this MSR.
2032	*
2033	* @param pVCpu The cross context virtual CPU structure.
2034	* @param pVmcsInfo The VMCS info. object.
2035	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2036	* @param idMsr The MSR value.
2037	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2038	* include both a read -and- a write permission!
2039	*
2040	* @sa CPUMGetVmxMsrPermission.
2041	* @remarks Can be called with interrupts disabled.
2042	*/
2043	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2044	{
2045	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2046	Assert(pbMsrBitmap);
2047	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2048
2049	/*
2050	* MSR-bitmap Layout:
2051	* Byte index MSR range Interpreted as
2052	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2053	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2054	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2055	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2056	*
2057	* A bit corresponding to an MSR within the above range causes a VM-exit
2058	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2059	* the MSR range, it always cause a VM-exit.
2060	*
2061	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2062	*/
2063	uint16_t const offBitmapRead = 0;
2064	uint16_t const offBitmapWrite = 0x800;
2065	uint16_t offMsr;
2066	int32_t iBit;
2067	if (idMsr <= UINT32_C(0x00001fff))
2068	{
2069	offMsr = 0;
2070	iBit = idMsr;
2071	}
2072	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2073	{
2074	offMsr = 0x400;
2075	iBit = idMsr - UINT32_C(0xc0000000);
2076	}
2077	else
2078	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2079
2080	/*
2081	* Set the MSR read permission.
2082	*/
2083	uint16_t const offMsrRead = offBitmapRead + offMsr;
2084	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2085	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2086	{
2087	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2088	bool const fClear = !fIsNstGstVmcs ? true
2089	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2090	#else
2091	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2092	bool const fClear = true;
2093	#endif
2094	if (fClear)
2095	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2096	}
2097	else
2098	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2099
2100	/*
2101	* Set the MSR write permission.
2102	*/
2103	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2104	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2105	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2106	{
2107	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2108	bool const fClear = !fIsNstGstVmcs ? true
2109	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2110	#else
2111	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2112	bool const fClear = true;
2113	#endif
2114	if (fClear)
2115	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2116	}
2117	else
2118	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2119	}
2120
2121
2122	/**
2123	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2124	* area.
2125	*
2126	* @returns VBox status code.
2127	* @param pVCpu The cross context virtual CPU structure.
2128	* @param pVmcsInfo The VMCS info. object.
2129	* @param cMsrs The number of MSRs.
2130	*/
2131	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2132	{
2133	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2134	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2135	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2136	{
2137	/* Commit the MSR counts to the VMCS and update the cache. */
2138	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2139	{
2140	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2141	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2142	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2143	AssertRCReturn(rc, rc);
2144
2145	pVmcsInfo->cEntryMsrLoad = cMsrs;
2146	pVmcsInfo->cExitMsrStore = cMsrs;
2147	pVmcsInfo->cExitMsrLoad = cMsrs;
2148	}
2149	return VINF_SUCCESS;
2150	}
2151
2152	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2153	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2154	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2155	}
2156
2157
2158	/**
2159	* Adds a new (or updates the value of an existing) guest/host MSR
2160	* pair to be swapped during the world-switch as part of the
2161	* auto-load/store MSR area in the VMCS.
2162	*
2163	* @returns VBox status code.
2164	* @param pVCpu The cross context virtual CPU structure.
2165	* @param pVmxTransient The VMX-transient structure.
2166	* @param idMsr The MSR.
2167	* @param uGuestMsrValue Value of the guest MSR.
2168	* @param fSetReadWrite Whether to set the guest read/write access of this
2169	* MSR (thus not causing a VM-exit).
2170	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2171	* necessary.
2172	*/
2173	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2174	bool fSetReadWrite, bool fUpdateHostMsr)
2175	{
2176	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2177	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2178	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2179	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2180	uint32_t i;
2181
2182	/* Paranoia. */
2183	Assert(pGuestMsrLoad);
2184
2185	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2186
2187	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2188	for (i = 0; i < cMsrs; i++)
2189	{
2190	if (pGuestMsrLoad[i].u32Msr == idMsr)
2191	break;
2192	}
2193
2194	bool fAdded = false;
2195	if (i == cMsrs)
2196	{
2197	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2198	++cMsrs;
2199	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2200	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2201
2202	/* Set the guest to read/write this MSR without causing VM-exits. */
2203	if ( fSetReadWrite
2204	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2205	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2206
2207	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2208	fAdded = true;
2209	}
2210
2211	/* Update the MSR value for the newly added or already existing MSR. */
2212	pGuestMsrLoad[i].u32Msr = idMsr;
2213	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2214
2215	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2216	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2217	{
2218	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2219	pGuestMsrStore[i].u32Msr = idMsr;
2220	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2221	}
2222
2223	/* Update the corresponding slot in the host MSR area. */
2224	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2225	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2226	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2227	pHostMsr[i].u32Msr = idMsr;
2228
2229	/*
2230	* Only if the caller requests to update the host MSR value AND we've newly added the
2231	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2232	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2233	*
2234	* We do this for performance reasons since reading MSRs may be quite expensive.
2235	*/
2236	if (fAdded)
2237	{
2238	if (fUpdateHostMsr)
2239	{
2240	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2241	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2242	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2243	}
2244	else
2245	{
2246	/* Someone else can do the work. */
2247	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2248	}
2249	}
2250	return VINF_SUCCESS;
2251	}
2252
2253
2254	/**
2255	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2256	* auto-load/store MSR area in the VMCS.
2257	*
2258	* @returns VBox status code.
2259	* @param pVCpu The cross context virtual CPU structure.
2260	* @param pVmxTransient The VMX-transient structure.
2261	* @param idMsr The MSR.
2262	*/
2263	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2264	{
2265	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2266	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2267	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2268	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2269
2270	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2271
2272	for (uint32_t i = 0; i < cMsrs; i++)
2273	{
2274	/* Find the MSR. */
2275	if (pGuestMsrLoad[i].u32Msr == idMsr)
2276	{
2277	/*
2278	* If it's the last MSR, we only need to reduce the MSR count.
2279	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2280	*/
2281	if (i < cMsrs - 1)
2282	{
2283	/* Remove it from the VM-entry MSR-load area. */
2284	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2285	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2286
2287	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2288	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2289	{
2290	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2291	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2292	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2293	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2294	}
2295
2296	/* Remove it from the VM-exit MSR-load area. */
2297	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2298	Assert(pHostMsr[i].u32Msr == idMsr);
2299	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2300	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2301	}
2302
2303	/* Reduce the count to reflect the removed MSR and bail. */
2304	--cMsrs;
2305	break;
2306	}
2307	}
2308
2309	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2310	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2311	{
2312	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2313	AssertRCReturn(rc, rc);
2314
2315	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2316	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2317	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2318
2319	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2320	return VINF_SUCCESS;
2321	}
2322
2323	return VERR_NOT_FOUND;
2324	}
2325
2326
2327	/**
2328	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2329	*
2330	* @returns @c true if found, @c false otherwise.
2331	* @param pVmcsInfo The VMCS info. object.
2332	* @param idMsr The MSR to find.
2333	*/
2334	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2335	{
2336	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2337	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2338	Assert(pMsrs);
2339	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2340	for (uint32_t i = 0; i < cMsrs; i++)
2341	{
2342	if (pMsrs[i].u32Msr == idMsr)
2343	return true;
2344	}
2345	return false;
2346	}
2347
2348
2349	/**
2350	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2351	*
2352	* @param pVCpu The cross context virtual CPU structure.
2353	* @param pVmcsInfo The VMCS info. object.
2354	*
2355	* @remarks No-long-jump zone!!!
2356	*/
2357	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2358	{
2359	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2360
2361	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2362	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2363	Assert(pHostMsrLoad);
2364	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2365	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2366	for (uint32_t i = 0; i < cMsrs; i++)
2367	{
2368	/*
2369	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2370	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2371	*/
2372	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2373	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2374	else
2375	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2376	}
2377	}
2378
2379
2380	/**
2381	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2382	* perform lazy restoration of the host MSRs while leaving VT-x.
2383	*
2384	* @param pVCpu The cross context virtual CPU structure.
2385	*
2386	* @remarks No-long-jump zone!!!
2387	*/
2388	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2389	{
2390	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2391
2392	/*
2393	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2394	*/
2395	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2396	{
2397	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2398	#if HC_ARCH_BITS == 64
2399	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2400	{
2401	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2402	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2403	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2404	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2405	}
2406	#endif
2407	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2408	}
2409	}
2410
2411
2412	/**
2413	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2414	* lazily while leaving VT-x.
2415	*
2416	* @returns true if it does, false otherwise.
2417	* @param pVCpu The cross context virtual CPU structure.
2418	* @param idMsr The MSR to check.
2419	*/
2420	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2421	{
2422	NOREF(pVCpu);
2423	#if HC_ARCH_BITS == 64
2424	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2425	{
2426	switch (idMsr)
2427	{
2428	case MSR_K8_LSTAR:
2429	case MSR_K6_STAR:
2430	case MSR_K8_SF_MASK:
2431	case MSR_K8_KERNEL_GS_BASE:
2432	return true;
2433	}
2434	}
2435	#else
2436	RT_NOREF(pVCpu, idMsr);
2437	#endif
2438	return false;
2439	}
2440
2441
2442	/**
2443	* Loads a set of guests MSRs to allow read/passthru to the guest.
2444	*
2445	* The name of this function is slightly confusing. This function does NOT
2446	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2447	* common prefix for functions dealing with "lazy restoration" of the shared
2448	* MSRs.
2449	*
2450	* @param pVCpu The cross context virtual CPU structure.
2451	*
2452	* @remarks No-long-jump zone!!!
2453	*/
2454	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2455	{
2456	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2457	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2458
2459	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2460	#if HC_ARCH_BITS == 64
2461	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2462	{
2463	/*
2464	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2465	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2466	* we can skip a few MSR writes.
2467	*
2468	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2469	* guest MSR values in the guest-CPU context might be different to what's currently
2470	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2471	* CPU, see @bugref{8728}.
2472	*/
2473	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2474	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2475	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2476	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2477	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2478	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2479	{
2480	#ifdef VBOX_STRICT
2481	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2482	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2483	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2484	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2485	#endif
2486	}
2487	else
2488	{
2489	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2490	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2491	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2492	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2493	}
2494	}
2495	#endif
2496	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2497	}
2498
2499
2500	/**
2501	* Performs lazy restoration of the set of host MSRs if they were previously
2502	* loaded with guest MSR values.
2503	*
2504	* @param pVCpu The cross context virtual CPU structure.
2505	*
2506	* @remarks No-long-jump zone!!!
2507	* @remarks The guest MSRs should have been saved back into the guest-CPU
2508	* context by hmR0VmxImportGuestState()!!!
2509	*/
2510	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2511	{
2512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2513	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2514
2515	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2516	{
2517	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2518	#if HC_ARCH_BITS == 64
2519	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2520	{
2521	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2522	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2523	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2524	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2525	}
2526	#endif
2527	}
2528	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2529	}
2530
2531
2532	/**
2533	* Verifies that our cached values of the VMCS fields are all consistent with
2534	* what's actually present in the VMCS.
2535	*
2536	* @returns VBox status code.
2537	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2538	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2539	* VMCS content. HMCPU error-field is
2540	* updated, see VMX_VCI_XXX.
2541	* @param pVCpu The cross context virtual CPU structure.
2542	* @param pVmcsInfo The VMCS info. object.
2543	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2544	*/
2545	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2546	{
2547	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2548
2549	uint32_t u32Val;
2550	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2551	AssertRCReturn(rc, rc);
2552	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2553	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2554	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2555	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2556
2557	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2558	AssertRCReturn(rc, rc);
2559	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2560	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2561	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2562	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2563
2564	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2565	AssertRCReturn(rc, rc);
2566	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2567	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2568	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2569	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2570
2571	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2572	AssertRCReturn(rc, rc);
2573	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2574	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2575	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2576	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2577
2578	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2579	{
2580	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2581	AssertRCReturn(rc, rc);
2582	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2583	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2584	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2585	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2586	}
2587
2588	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2589	AssertRCReturn(rc, rc);
2590	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2591	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2592	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2593	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2594
2595	uint64_t u64Val;
2596	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2597	AssertRCReturn(rc, rc);
2598	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2599	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2600	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2601	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2602
2603	NOREF(pcszVmcs);
2604	return VINF_SUCCESS;
2605	}
2606
2607
2608	#ifdef VBOX_STRICT
2609	/**
2610	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2611	*
2612	* @param pVCpu The cross context virtual CPU structure.
2613	* @param pVmcsInfo The VMCS info. object.
2614	*/
2615	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2616	{
2617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2618
2619	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2620	{
2621	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2622	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2623	uint64_t uVmcsEferMsrVmcs;
2624	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2625	AssertRC(rc);
2626
2627	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2628	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2629	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2630	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2631	}
2632	}
2633
2634
2635	/**
2636	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2637	* VMCS are correct.
2638	*
2639	* @param pVCpu The cross context virtual CPU structure.
2640	* @param pVmcsInfo The VMCS info. object.
2641	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2642	*/
2643	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2644	{
2645	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2646
2647	/* Read the various MSR-area counts from the VMCS. */
2648	uint32_t cEntryLoadMsrs;
2649	uint32_t cExitStoreMsrs;
2650	uint32_t cExitLoadMsrs;
2651	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2652	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2653	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2654
2655	/* Verify all the MSR counts are the same. */
2656	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2657	Assert(cExitStoreMsrs == cExitLoadMsrs);
2658	uint32_t const cMsrs = cExitLoadMsrs;
2659
2660	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2661	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2662
2663	/* Verify the MSR counts are within the allocated page size. */
2664	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2665
2666	/* Verify the relevant contents of the MSR areas match. */
2667	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2668	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2669	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2670	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2671	for (uint32_t i = 0; i < cMsrs; i++)
2672	{
2673	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2674	if (fSeparateExitMsrStorePage)
2675	{
2676	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2677	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2678	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2679	}
2680
2681	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2682	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2683	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2684
2685	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2686	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2687	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2688	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2689
2690	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2691	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2692	if (fIsEferMsr)
2693	{
2694	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2695	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2696	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2697	}
2698
2699	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2700	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2701	{
2702	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2703	if (fIsEferMsr)
2704	{
2705	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2706	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2707	}
2708	else
2709	{
2710	if (!fIsNstGstVmcs)
2711	{
2712	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2713	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2714	}
2715	else
2716	{
2717	/*
2718	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2719	* execute a nested-guest with MSR passthrough.
2720	*
2721	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2722	* allow passthrough too.
2723	*/
2724	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2725	Assert(pvMsrBitmapNstGst);
2726	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2727	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2728	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2729	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2730	}
2731	}
2732	}
2733
2734	/* Move to the next MSR. */
2735	pHostMsrLoad++;
2736	pGuestMsrLoad++;
2737	pGuestMsrStore++;
2738	}
2739	}
2740	#endif /* VBOX_STRICT */
2741
2742
2743	/**
2744	* Flushes the TLB using EPT.
2745	*
2746	* @returns VBox status code.
2747	* @param pVCpu The cross context virtual CPU structure of the calling
2748	* EMT. Can be NULL depending on @a enmTlbFlush.
2749	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2750	* enmTlbFlush.
2751	* @param enmTlbFlush Type of flush.
2752	*
2753	* @remarks Caller is responsible for making sure this function is called only
2754	* when NestedPaging is supported and providing @a enmTlbFlush that is
2755	* supported by the CPU.
2756	* @remarks Can be called with interrupts disabled.
2757	*/
2758	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2759	{
2760	uint64_t au64Descriptor[2];
2761	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2762	au64Descriptor[0] = 0;
2763	else
2764	{
2765	Assert(pVCpu);
2766	Assert(pVmcsInfo);
2767	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2768	}
2769	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2770
2771	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2772	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2773
2774	if ( RT_SUCCESS(rc)
2775	&& pVCpu)
2776	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2777	}
2778
2779
2780	/**
2781	* Flushes the TLB using VPID.
2782	*
2783	* @returns VBox status code.
2784	* @param pVCpu The cross context virtual CPU structure of the calling
2785	* EMT. Can be NULL depending on @a enmTlbFlush.
2786	* @param enmTlbFlush Type of flush.
2787	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2788	* on @a enmTlbFlush).
2789	*
2790	* @remarks Can be called with interrupts disabled.
2791	*/
2792	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2793	{
2794	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2795
2796	uint64_t au64Descriptor[2];
2797	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2798	{
2799	au64Descriptor[0] = 0;
2800	au64Descriptor[1] = 0;
2801	}
2802	else
2803	{
2804	AssertPtr(pVCpu);
2805	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2806	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2807	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2808	au64Descriptor[1] = GCPtr;
2809	}
2810
2811	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2812	AssertMsg(rc == VINF_SUCCESS,
2813	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2814
2815	if ( RT_SUCCESS(rc)
2816	&& pVCpu)
2817	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2818	NOREF(rc);
2819	}
2820
2821
2822	/**
2823	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2824	* otherwise there is nothing really to invalidate.
2825	*
2826	* @returns VBox status code.
2827	* @param pVCpu The cross context virtual CPU structure.
2828	* @param GCVirt Guest virtual address of the page to invalidate.
2829	*/
2830	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2831	{
2832	AssertPtr(pVCpu);
2833	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2834
2835	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2836	{
2837	/*
2838	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2839	* the EPT case. See @bugref{6043} and @bugref{6177}.
2840	*
2841	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2842	* as this function maybe called in a loop with individual addresses.
2843	*/
2844	PVM pVM = pVCpu->CTX_SUFF(pVM);
2845	if (pVM->hm.s.vmx.fVpid)
2846	{
2847	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2848
2849	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2850	/*
2851	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2852	* where executing INVVPID outside 64-bit mode does not flush translations of
2853	* 64-bit linear addresses, see @bugref{6208#c72}.
2854	*/
2855	if (RT_HI_U32(GCVirt))
2856	fVpidFlush = false;
2857	#endif
2858
2859	if (fVpidFlush)
2860	{
2861	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2862	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2863	}
2864	else
2865	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2866	}
2867	else if (pVM->hm.s.fNestedPaging)
2868	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2869	}
2870
2871	return VINF_SUCCESS;
2872	}
2873
2874
2875	/**
2876	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2877	* case where neither EPT nor VPID is supported by the CPU.
2878	*
2879	* @param pHostCpu The HM physical-CPU structure.
2880	* @param pVCpu The cross context virtual CPU structure.
2881	*
2882	* @remarks Called with interrupts disabled.
2883	*/
2884	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2885	{
2886	AssertPtr(pVCpu);
2887	AssertPtr(pHostCpu);
2888
2889	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2890
2891	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2892	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2893	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2894	pVCpu->hm.s.fForceTLBFlush = false;
2895	return;
2896	}
2897
2898
2899	/**
2900	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2901	*
2902	* @param pHostCpu The HM physical-CPU structure.
2903	* @param pVCpu The cross context virtual CPU structure.
2904	* @param pVmcsInfo The VMCS info. object.
2905	*
2906	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2907	* nomenclature. The reason is, to avoid confusion in compare statements
2908	* since the host-CPU copies are named "ASID".
2909	*
2910	* @remarks Called with interrupts disabled.
2911	*/
2912	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2913	{
2914	#ifdef VBOX_WITH_STATISTICS
2915	bool fTlbFlushed = false;
2916	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2917	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2918	if (!fTlbFlushed) \
2919	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2920	} while (0)
2921	#else
2922	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2923	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2924	#endif
2925
2926	AssertPtr(pVCpu);
2927	AssertPtr(pHostCpu);
2928	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2929
2930	PVM pVM = pVCpu->CTX_SUFF(pVM);
2931	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2932	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2933	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2934
2935	/*
2936	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2937	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2938	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2939	* cannot reuse the current ASID anymore.
2940	*/
2941	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2942	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2943	{
2944	++pHostCpu->uCurrentAsid;
2945	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2946	{
2947	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2948	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2949	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2950	}
2951
2952	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2953	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2954	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2955
2956	/*
2957	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2958	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2959	*/
2960	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2961	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2962	HMVMX_SET_TAGGED_TLB_FLUSHED();
2963	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2964	}
2965	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2966	{
2967	/*
2968	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2969	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2970	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2971	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2972	* mappings, see @bugref{6568}.
2973	*
2974	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2975	*/
2976	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2977	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2978	HMVMX_SET_TAGGED_TLB_FLUSHED();
2979	}
2980
2981	pVCpu->hm.s.fForceTLBFlush = false;
2982	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2983
2984	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2985	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2986	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2987	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2988	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2989	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2990	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2991	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2992	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2993
2994	/* Update VMCS with the VPID. */
2995	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2996	AssertRC(rc);
2997
2998	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2999	}
3000
3001
3002	/**
3003	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3004	*
3005	* @param pHostCpu The HM physical-CPU structure.
3006	* @param pVCpu The cross context virtual CPU structure.
3007	* @param pVmcsInfo The VMCS info. object.
3008	*
3009	* @remarks Called with interrupts disabled.
3010	*/
3011	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3012	{
3013	AssertPtr(pVCpu);
3014	AssertPtr(pHostCpu);
3015	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3016	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3017	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3018
3019	/*
3020	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3021	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3022	*/
3023	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3024	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3025	{
3026	pVCpu->hm.s.fForceTLBFlush = true;
3027	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3028	}
3029
3030	/* Check for explicit TLB flushes. */
3031	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3032	{
3033	pVCpu->hm.s.fForceTLBFlush = true;
3034	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3035	}
3036
3037	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3038	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3039
3040	if (pVCpu->hm.s.fForceTLBFlush)
3041	{
3042	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3043	pVCpu->hm.s.fForceTLBFlush = false;
3044	}
3045	}
3046
3047
3048	/**
3049	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3050	*
3051	* @param pHostCpu The HM physical-CPU structure.
3052	* @param pVCpu The cross context virtual CPU structure.
3053	*
3054	* @remarks Called with interrupts disabled.
3055	*/
3056	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3057	{
3058	AssertPtr(pVCpu);
3059	AssertPtr(pHostCpu);
3060	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3061	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3062	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3063
3064	/*
3065	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3066	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3067	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3068	* cannot reuse the current ASID anymore.
3069	*/
3070	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3071	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3072	{
3073	pVCpu->hm.s.fForceTLBFlush = true;
3074	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3075	}
3076
3077	/* Check for explicit TLB flushes. */
3078	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3079	{
3080	/*
3081	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3082	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3083	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3084	* include fExplicitFlush's too) - an obscure corner case.
3085	*/
3086	pVCpu->hm.s.fForceTLBFlush = true;
3087	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3088	}
3089
3090	PVM pVM = pVCpu->CTX_SUFF(pVM);
3091	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3092	if (pVCpu->hm.s.fForceTLBFlush)
3093	{
3094	++pHostCpu->uCurrentAsid;
3095	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3096	{
3097	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3098	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3099	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3100	}
3101
3102	pVCpu->hm.s.fForceTLBFlush = false;
3103	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3104	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3105	if (pHostCpu->fFlushAsidBeforeUse)
3106	{
3107	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3108	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3109	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3110	{
3111	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3112	pHostCpu->fFlushAsidBeforeUse = false;
3113	}
3114	else
3115	{
3116	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3117	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3118	}
3119	}
3120	}
3121
3122	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3123	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3124	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3125	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3126	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3127	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3128	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3129
3130	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3131	AssertRC(rc);
3132	}
3133
3134
3135	/**
3136	* Flushes the guest TLB entry based on CPU capabilities.
3137	*
3138	* @param pHostCpu The HM physical-CPU structure.
3139	* @param pVCpu The cross context virtual CPU structure.
3140	* @param pVmcsInfo The VMCS info. object.
3141	*
3142	* @remarks Called with interrupts disabled.
3143	*/
3144	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3145	{
3146	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3147	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3148	#endif
3149	PVM pVM = pVCpu->CTX_SUFF(pVM);
3150	switch (pVM->hm.s.vmx.enmTlbFlushType)
3151	{
3152	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3153	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3154	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3155	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3156	default:
3157	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3158	break;
3159	}
3160	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3161	}
3162
3163
3164	/**
3165	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3166	* TLB entries from the host TLB before VM-entry.
3167	*
3168	* @returns VBox status code.
3169	* @param pVM The cross context VM structure.
3170	*/
3171	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3172	{
3173	/*
3174	* Determine optimal flush type for nested paging.
3175	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3176	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3177	*/
3178	if (pVM->hm.s.fNestedPaging)
3179	{
3180	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3181	{
3182	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3183	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3184	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3185	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3186	else
3187	{
3188	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3189	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3190	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3191	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3192	}
3193
3194	/* Make sure the write-back cacheable memory type for EPT is supported. */
3195	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3196	{
3197	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3198	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3199	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3200	}
3201
3202	/* EPT requires a page-walk length of 4. */
3203	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3204	{
3205	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3206	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3207	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3208	}
3209	}
3210	else
3211	{
3212	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3213	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3214	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3215	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3216	}
3217	}
3218
3219	/*
3220	* Determine optimal flush type for VPID.
3221	*/
3222	if (pVM->hm.s.vmx.fVpid)
3223	{
3224	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3225	{
3226	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3227	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3228	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3229	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3230	else
3231	{
3232	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3233	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3234	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3235	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3236	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3237	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3238	pVM->hm.s.vmx.fVpid = false;
3239	}
3240	}
3241	else
3242	{
3243	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3244	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3245	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3246	pVM->hm.s.vmx.fVpid = false;
3247	}
3248	}
3249
3250	/*
3251	* Setup the handler for flushing tagged-TLBs.
3252	*/
3253	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3254	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3255	else if (pVM->hm.s.fNestedPaging)
3256	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3257	else if (pVM->hm.s.vmx.fVpid)
3258	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3259	else
3260	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3261	return VINF_SUCCESS;
3262	}
3263
3264
3265	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3266	/**
3267	* Sets up the shadow VMCS fields arrays.
3268	*
3269	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3270	* executing the guest.
3271	*
3272	* @returns VBox status code.
3273	* @param pVM The cross context VM structure.
3274	*/
3275	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3276	{
3277	/*
3278	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3279	* when the host does not support it.
3280	*/
3281	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3282	if ( !fGstVmwriteAll
3283	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3284	{ /* likely. */ }
3285	else
3286	{
3287	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3288	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3289	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3290	}
3291
3292	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3293	uint32_t cRwFields = 0;
3294	uint32_t cRoFields = 0;
3295	for (uint32_t i = 0; i < cVmcsFields; i++)
3296	{
3297	VMXVMCSFIELD VmcsField;
3298	VmcsField.u = g_aVmcsFields[i];
3299
3300	/*
3301	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3302	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3303	* in the shadow VMCS fields array as they would be redundant.
3304	*
3305	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3306	* we must not include it in the shadow VMCS fields array. Guests attempting to
3307	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3308	* the required behavior.
3309	*/
3310	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3311	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3312	{
3313	/*
3314	* Read-only fields are placed in a separate array so that while syncing shadow
3315	* VMCS fields later (which is more performance critical) we can avoid branches.
3316	*
3317	* However, if the guest can write to all fields (including read-only fields),
3318	* we treat it a as read/write field. Otherwise, writing to these fields would
3319	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3320	*/
3321	if ( fGstVmwriteAll
3322	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3323	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3324	else
3325	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3326	}
3327	}
3328
3329	/* Update the counts. */
3330	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3331	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3332	return VINF_SUCCESS;
3333	}
3334
3335
3336	/**
3337	* Sets up the VMREAD and VMWRITE bitmaps.
3338	*
3339	* @param pVM The cross context VM structure.
3340	*/
3341	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3342	{
3343	/*
3344	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3345	*/
3346	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3347	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3348	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3349	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3350	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3351
3352	/*
3353	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3354	* VMREAD and VMWRITE bitmaps.
3355	*/
3356	{
3357	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3358	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3359	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3360	{
3361	uint32_t const uVmcsField = paShadowVmcsFields[i];
3362	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3363	Assert(uVmcsField >> 3 < cbBitmap);
3364	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3365	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3366	}
3367	}
3368
3369	/*
3370	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3371	* if the host supports VMWRITE to all supported VMCS fields.
3372	*/
3373	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3374	{
3375	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3376	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3377	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3378	{
3379	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3380	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3381	Assert(uVmcsField >> 3 < cbBitmap);
3382	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3383	}
3384	}
3385	}
3386	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3387
3388
3389	/**
3390	* Sets up the virtual-APIC page address for the VMCS.
3391	*
3392	* @returns VBox status code.
3393	* @param pVCpu The cross context virtual CPU structure.
3394	* @param pVmcsInfo The VMCS info. object.
3395	*/
3396	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3397	{
3398	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3399	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3400	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3401	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3402	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3403	}
3404
3405
3406	/**
3407	* Sets up the MSR-bitmap address for the VMCS.
3408	*
3409	* @returns VBox status code.
3410	* @param pVCpu The cross context virtual CPU structure.
3411	* @param pVmcsInfo The VMCS info. object.
3412	*/
3413	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3414	{
3415	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3416	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3417	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3418	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3419	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3420	}
3421
3422
3423	/**
3424	* Sets up the APIC-access page address for the VMCS.
3425	*
3426	* @returns VBox status code.
3427	* @param pVCpu The cross context virtual CPU structure.
3428	*/
3429	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3430	{
3431	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3432	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3433	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3434	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3435	}
3436
3437
3438	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3439	/**
3440	* Sets up the VMREAD bitmap address for the VMCS.
3441	*
3442	* @returns VBox status code.
3443	* @param pVCpu The cross context virtual CPU structure.
3444	*/
3445	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3446	{
3447	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3448	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3449	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3450	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3451	}
3452
3453
3454	/**
3455	* Sets up the VMWRITE bitmap address for the VMCS.
3456	*
3457	* @returns VBox status code.
3458	* @param pVCpu The cross context virtual CPU structure.
3459	*/
3460	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3461	{
3462	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3463	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3464	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3465	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3466	}
3467	#endif
3468
3469
3470	/**
3471	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3472	* in the VMCS.
3473	*
3474	* @returns VBox status code.
3475	* @param pVCpu The cross context virtual CPU structure.
3476	* @param pVmcsInfo The VMCS info. object.
3477	*/
3478	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3479	{
3480	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3481
3482	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3483	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3484	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3485
3486	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3487	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3488	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3489
3490	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3491	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3492	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3493
3494	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3495	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3496	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3497	AssertRCReturn(rc, rc);
3498	return VINF_SUCCESS;
3499	}
3500
3501
3502	/**
3503	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3504	*
3505	* @param pVCpu The cross context virtual CPU structure.
3506	* @param pVmcsInfo The VMCS info. object.
3507	*/
3508	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3509	{
3510	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3511
3512	/*
3513	* The guest can access the following MSRs (read, write) without causing
3514	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3515	*/
3516	PVM pVM = pVCpu->CTX_SUFF(pVM);
3517	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3518	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3519	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3520	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3521	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3522
3523	/*
3524	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3525	* associated with then. We never need to intercept access (writes need to be
3526	* executed without causing a VM-exit, reads will #GP fault anyway).
3527	*
3528	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3529	* read/write them. We swap the the guest/host MSR value using the
3530	* auto-load/store MSR area.
3531	*/
3532	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3533	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3534	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3535	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3536	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3537	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3538
3539	#if HC_ARCH_BITS == 64
3540	/*
3541	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3542	* required for 64-bit guests.
3543	*/
3544	if (pVM->hm.s.fAllow64BitGuests)
3545	{
3546	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3547	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3548	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3549	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3550	}
3551	#endif
3552
3553	/*
3554	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3555	*/
3556	#ifdef VBOX_STRICT
3557	Assert(pVmcsInfo->pvMsrBitmap);
3558	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3559	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3560	#endif
3561	}
3562
3563
3564	/**
3565	* Sets up pin-based VM-execution controls in the VMCS.
3566	*
3567	* @returns VBox status code.
3568	* @param pVCpu The cross context virtual CPU structure.
3569	* @param pVmcsInfo The VMCS info. object.
3570	*/
3571	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3572	{
3573	PVM pVM = pVCpu->CTX_SUFF(pVM);
3574	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3575	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3576
3577	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3578	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3579
3580	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3581	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3582
3583	/* Enable the VMX-preemption timer. */
3584	if (pVM->hm.s.vmx.fUsePreemptTimer)
3585	{
3586	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3587	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3588	}
3589
3590	#if 0
3591	/* Enable posted-interrupt processing. */
3592	if (pVM->hm.s.fPostedIntrs)
3593	{
3594	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3595	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3596	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3597	}
3598	#endif
3599
3600	if ((fVal & fZap) != fVal)
3601	{
3602	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3603	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3604	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3605	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3606	}
3607
3608	/* Commit it to the VMCS and update our cache. */
3609	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3610	AssertRCReturn(rc, rc);
3611	pVmcsInfo->u32PinCtls = fVal;
3612
3613	return VINF_SUCCESS;
3614	}
3615
3616
3617	/**
3618	* Sets up secondary processor-based VM-execution controls in the VMCS.
3619	*
3620	* @returns VBox status code.
3621	* @param pVCpu The cross context virtual CPU structure.
3622	* @param pVmcsInfo The VMCS info. object.
3623	*/
3624	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3625	{
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3628	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3629
3630	/* WBINVD causes a VM-exit. */
3631	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3632	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3633
3634	/* Enable EPT (aka nested-paging). */
3635	if (pVM->hm.s.fNestedPaging)
3636	fVal \|= VMX_PROC_CTLS2_EPT;
3637
3638	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3639	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3640	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3641	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3642	fVal \|= VMX_PROC_CTLS2_INVPCID;
3643
3644	/* Enable VPID. */
3645	if (pVM->hm.s.vmx.fVpid)
3646	fVal \|= VMX_PROC_CTLS2_VPID;
3647
3648	/* Enable unrestricted guest execution. */
3649	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3650	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3651
3652	#if 0
3653	if (pVM->hm.s.fVirtApicRegs)
3654	{
3655	/* Enable APIC-register virtualization. */
3656	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3657	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3658
3659	/* Enable virtual-interrupt delivery. */
3660	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3661	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3662	}
3663	#endif
3664
3665	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3666	where the TPR shadow resides. */
3667	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3668	* done dynamically. */
3669	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3670	{
3671	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3672	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3673	AssertRCReturn(rc, rc);
3674	}
3675
3676	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3677	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3678	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3679	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3680	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3681
3682	/* Enable Pause-Loop exiting. */
3683	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3684	&& pVM->hm.s.vmx.cPleGapTicks
3685	&& pVM->hm.s.vmx.cPleWindowTicks)
3686	{
3687	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3688
3689	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3690	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3691	AssertRCReturn(rc, rc);
3692	}
3693
3694	if ((fVal & fZap) != fVal)
3695	{
3696	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3697	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3698	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3699	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3700	}
3701
3702	/* Commit it to the VMCS and update our cache. */
3703	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3704	AssertRCReturn(rc, rc);
3705	pVmcsInfo->u32ProcCtls2 = fVal;
3706
3707	return VINF_SUCCESS;
3708	}
3709
3710
3711	/**
3712	* Sets up processor-based VM-execution controls in the VMCS.
3713	*
3714	* @returns VBox status code.
3715	* @param pVCpu The cross context virtual CPU structure.
3716	* @param pVmcsInfo The VMCS info. object.
3717	*/
3718	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3719	{
3720	PVM pVM = pVCpu->CTX_SUFF(pVM);
3721
3722	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3723	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3724
3725	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3726	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3727	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3728	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3729	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3730	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3731	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3732
3733	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3734	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3735	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3736	{
3737	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3738	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3739	}
3740
3741	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3742	if (!pVM->hm.s.fNestedPaging)
3743	{
3744	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3745	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3746	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3747	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3748	}
3749
3750	/* Use TPR shadowing if supported by the CPU. */
3751	if ( PDMHasApic(pVM)
3752	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3753	{
3754	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3755	/* CR8 writes cause a VM-exit based on TPR threshold. */
3756	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3757	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3758	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3759	AssertRCReturn(rc, rc);
3760	}
3761	else
3762	{
3763	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3764	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3765	if (pVM->hm.s.fAllow64BitGuests)
3766	{
3767	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3768	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3769	}
3770	}
3771
3772	/* Use MSR-bitmaps if supported by the CPU. */
3773	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3774	{
3775	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3776	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3777	AssertRCReturn(rc, rc);
3778	}
3779
3780	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3781	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3782	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3783
3784	if ((fVal & fZap) != fVal)
3785	{
3786	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3787	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3788	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3789	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3790	}
3791
3792	/* Commit it to the VMCS and update our cache. */
3793	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3794	AssertRCReturn(rc, rc);
3795	pVmcsInfo->u32ProcCtls = fVal;
3796
3797	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3798	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3799	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3800
3801	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3802	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3803	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3804
3805	/* Sanity check, should not really happen. */
3806	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3807	{ /* likely */ }
3808	else
3809	{
3810	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3811	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3812	}
3813
3814	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3815	return VINF_SUCCESS;
3816	}
3817
3818
3819	/**
3820	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3821	* Processor-based VM-execution) control fields in the VMCS.
3822	*
3823	* @returns VBox status code.
3824	* @param pVCpu The cross context virtual CPU structure.
3825	* @param pVmcsInfo The VMCS info. object.
3826	*/
3827	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3828	{
3829	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3830	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3831	{
3832	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3833	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3834	if (RT_SUCCESS(rc))
3835	{ /* likely */ }
3836	else
3837	{
3838	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3839	return rc;
3840	}
3841	}
3842	#endif
3843
3844	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3845	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3846	if (RT_SUCCESS(rc))
3847	{
3848	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3849	if (RT_SUCCESS(rc))
3850	{
3851	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3852	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3853	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3854	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3855	if (RT_SUCCESS(rc))
3856	{
3857	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3858	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3859	return VINF_SUCCESS;
3860	}
3861	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3862	}
3863	else
3864	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3865	}
3866	else
3867	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3868	return rc;
3869	}
3870
3871
3872	/**
3873	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3874	*
3875	* We shall setup those exception intercepts that don't change during the
3876	* lifetime of the VM here. The rest are done dynamically while loading the
3877	* guest state.
3878	*
3879	* @returns VBox status code.
3880	* @param pVCpu The cross context virtual CPU structure.
3881	* @param pVmcsInfo The VMCS info. object.
3882	*/
3883	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3884	{
3885	/*
3886	* The following exceptions are always intercepted:
3887	*
3888	* #AC - To prevent the guest from hanging the CPU.
3889	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3890	* recursive #DBs can cause a CPU hang.
3891	* #PF - To sync our shadow page tables when nested-paging is not used.
3892	*/
3893	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3894	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3895	\| RT_BIT(X86_XCPT_DB)
3896	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3897
3898	/* Commit it to the VMCS. */
3899	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3900	AssertRCReturn(rc, rc);
3901
3902	/* Update our cache of the exception bitmap. */
3903	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3904	return VINF_SUCCESS;
3905	}
3906
3907
3908	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3909	/**
3910	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3911	*
3912	* @returns VBox status code.
3913	* @param pVCpu The cross context virtual CPU structure.
3914	* @param pVmcsInfo The VMCS info. object.
3915	*/
3916	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3917	{
3918	PVM pVM = pVCpu->CTX_SUFF(pVM);
3919	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3920	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3921	if (RT_SUCCESS(rc))
3922	{
3923	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3924	if (RT_SUCCESS(rc))
3925	{
3926	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3927	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3928	if (RT_SUCCESS(rc))
3929	{
3930	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3931	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3932	if (RT_SUCCESS(rc))
3933	return VINF_SUCCESS;
3934
3935	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3936	}
3937	else
3938	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3939	}
3940	else
3941	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3942	}
3943	else
3944	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3945
3946	return rc;
3947	}
3948	#endif
3949
3950
3951	/**
3952	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3953	* VMX.
3954	*
3955	* @returns VBox status code.
3956	* @param pVCpu The cross context virtual CPU structure.
3957	* @param pVmcsInfo The VMCS info. object.
3958	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3959	*/
3960	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3961	{
3962	Assert(pVmcsInfo->pvVmcs);
3963	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3964
3965	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3966	PVM pVM = pVCpu->CTX_SUFF(pVM);
3967	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3968	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3969
3970	LogFlowFunc(("\n"));
3971
3972	/*
3973	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3974	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3975	*/
3976	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3977	if (RT_SUCCESS(rc))
3978	{
3979	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3980	if (RT_SUCCESS(rc))
3981	{
3982	if (!fIsNstGstVmcs)
3983	{
3984	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3985	if (RT_SUCCESS(rc))
3986	{
3987	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3988	if (RT_SUCCESS(rc))
3989	{
3990	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3991	if (RT_SUCCESS(rc))
3992	{
3993	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3994	if (RT_SUCCESS(rc))
3995	{
3996	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3997	/*
3998	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
3999	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4000	* making it fit for use when VMCS shadowing is later enabled.
4001	*/
4002	if (pVmcsInfo->pvShadowVmcs)
4003	{
4004	VMXVMCSREVID VmcsRevId;
4005	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4006	VmcsRevId.n.fIsShadowVmcs = 1;
4007	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4008	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4009	if (RT_SUCCESS(rc))
4010	{ /* likely */ }
4011	else
4012	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4013	}
4014	#endif
4015	}
4016	else
4017	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4018	}
4019	else
4020	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4021	}
4022	else
4023	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4024	}
4025	else
4026	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4027	}
4028	else
4029	{
4030	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4031	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4032	if (RT_SUCCESS(rc))
4033	{ /* likely */ }
4034	else
4035	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4036	#else
4037	AssertFailed();
4038	#endif
4039	}
4040	}
4041	else
4042	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4043	}
4044	else
4045	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4046
4047	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4048	if (RT_SUCCESS(rc))
4049	{
4050	rc = hmR0VmxClearVmcs(pVmcsInfo);
4051	if (RT_SUCCESS(rc))
4052	{ /* likely */ }
4053	else
4054	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4055	}
4056
4057	/*
4058	* Update the last-error record both for failures and success, so we
4059	* can propagate the status code back to ring-3 for diagnostics.
4060	*/
4061	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4062	NOREF(pszVmcs);
4063	return rc;
4064	}
4065
4066
4067	/**
4068	* Does global VT-x initialization (called during module initialization).
4069	*
4070	* @returns VBox status code.
4071	*/
4072	VMMR0DECL(int) VMXR0GlobalInit(void)
4073	{
4074	#ifdef HMVMX_USE_FUNCTION_TABLE
4075	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4076	# ifdef VBOX_STRICT
4077	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4078	Assert(g_apfnVMExitHandlers[i]);
4079	# endif
4080	#endif
4081	return VINF_SUCCESS;
4082	}
4083
4084
4085	/**
4086	* Does global VT-x termination (called during module termination).
4087	*/
4088	VMMR0DECL(void) VMXR0GlobalTerm()
4089	{
4090	/* Nothing to do currently. */
4091	}
4092
4093
4094	/**
4095	* Sets up and activates VT-x on the current CPU.
4096	*
4097	* @returns VBox status code.
4098	* @param pHostCpu The HM physical-CPU structure.
4099	* @param pVM The cross context VM structure. Can be
4100	* NULL after a host resume operation.
4101	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4102	* fEnabledByHost is @c true).
4103	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4104	* @a fEnabledByHost is @c true).
4105	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4106	* enable VT-x on the host.
4107	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4108	*/
4109	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4110	PCSUPHWVIRTMSRS pHwvirtMsrs)
4111	{
4112	AssertPtr(pHostCpu);
4113	AssertPtr(pHwvirtMsrs);
4114	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4115
4116	/* Enable VT-x if it's not already enabled by the host. */
4117	if (!fEnabledByHost)
4118	{
4119	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4120	if (RT_FAILURE(rc))
4121	return rc;
4122	}
4123
4124	/*
4125	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4126	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4127	* invalidated when flushing by VPID.
4128	*/
4129	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4130	{
4131	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4132	pHostCpu->fFlushAsidBeforeUse = false;
4133	}
4134	else
4135	pHostCpu->fFlushAsidBeforeUse = true;
4136
4137	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4138	++pHostCpu->cTlbFlushes;
4139
4140	return VINF_SUCCESS;
4141	}
4142
4143
4144	/**
4145	* Deactivates VT-x on the current CPU.
4146	*
4147	* @returns VBox status code.
4148	* @param pvCpuPage Pointer to the VMXON region.
4149	* @param HCPhysCpuPage Physical address of the VMXON region.
4150	*
4151	* @remarks This function should never be called when SUPR0EnableVTx() or
4152	* similar was used to enable VT-x on the host.
4153	*/
4154	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4155	{
4156	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4157
4158	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4159	return hmR0VmxLeaveRootMode();
4160	}
4161
4162
4163	/**
4164	* Does per-VM VT-x initialization.
4165	*
4166	* @returns VBox status code.
4167	* @param pVM The cross context VM structure.
4168	*/
4169	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4170	{
4171	AssertPtr(pVM);
4172	LogFlowFunc(("pVM=%p\n", pVM));
4173
4174	int rc = hmR0VmxStructsAlloc(pVM);
4175	if (RT_FAILURE(rc))
4176	{
4177	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4178	return rc;
4179	}
4180
4181	return VINF_SUCCESS;
4182	}
4183
4184
4185	/**
4186	* Does per-VM VT-x termination.
4187	*
4188	* @returns VBox status code.
4189	* @param pVM The cross context VM structure.
4190	*/
4191	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4192	{
4193	AssertPtr(pVM);
4194	LogFlowFunc(("pVM=%p\n", pVM));
4195
4196	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4197	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4198	{
4199	Assert(pVM->hm.s.vmx.pvScratch);
4200	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4201	}
4202	#endif
4203	hmR0VmxStructsFree(pVM);
4204	return VINF_SUCCESS;
4205	}
4206
4207
4208	/**
4209	* Sets up the VM for execution using hardware-assisted VMX.
4210	* This function is only called once per-VM during initialization.
4211	*
4212	* @returns VBox status code.
4213	* @param pVM The cross context VM structure.
4214	*/
4215	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4216	{
4217	AssertPtr(pVM);
4218	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4219
4220	LogFlowFunc(("pVM=%p\n", pVM));
4221
4222	/*
4223	* At least verify if VMX is enabled, since we can't check if we're in
4224	* VMX root mode or not without causing a #GP.
4225	*/
4226	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4227	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4228	{ /* likely */ }
4229	else
4230	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4231
4232	/*
4233	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4234	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4235	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4236	*/
4237	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4238	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4239	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4240	{
4241	LogRelFunc(("Invalid real-on-v86 state.\n"));
4242	return VERR_INTERNAL_ERROR;
4243	}
4244
4245	/* Initialize these always, see hmR3InitFinalizeR0().*/
4246	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4247	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4248
4249	/* Setup the tagged-TLB flush handlers. */
4250	int rc = hmR0VmxSetupTaggedTlb(pVM);
4251	if (RT_FAILURE(rc))
4252	{
4253	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4254	return rc;
4255	}
4256
4257	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4258	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4259	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4260	{
4261	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4262	if (RT_SUCCESS(rc))
4263	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4264	else
4265	{
4266	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4267	return rc;
4268	}
4269	}
4270	#endif
4271
4272	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4273	{
4274	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4275	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4276
4277	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4278	if (RT_SUCCESS(rc))
4279	{
4280	#if HC_ARCH_BITS == 32
4281	hmR0VmxInitVmcsReadCache(pVCpu);
4282	#endif
4283	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4284	if (pVM->cpum.ro.GuestFeatures.fVmx)
4285	{
4286	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4287	if (RT_SUCCESS(rc))
4288	{ /* likely */ }
4289	else
4290	{
4291	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4292	return rc;
4293	}
4294	}
4295	#endif
4296	}
4297	else
4298	{
4299	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4300	return rc;
4301	}
4302	}
4303
4304	return VINF_SUCCESS;
4305	}
4306
4307
4308	#if HC_ARCH_BITS == 32
4309	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4310	/**
4311	* Check if guest state allows safe use of 32-bit switcher again.
4312	*
4313	* Segment bases and protected mode structures must be 32-bit addressable
4314	* because the 32-bit switcher will ignore high dword when writing these VMCS
4315	* fields. See @bugref{8432} for details.
4316	*
4317	* @returns true if safe, false if must continue to use the 64-bit switcher.
4318	* @param pCtx Pointer to the guest-CPU context.
4319	*
4320	* @remarks No-long-jump zone!!!
4321	*/
4322	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4323	{
4324	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4325	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4326	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4327	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4328	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4329	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4330	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4331	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4332	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4333	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4334
4335	/* All good, bases are 32-bit. */
4336	return true;
4337	}
4338	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4339
4340	# ifdef VBOX_STRICT
4341	static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4342	{
4343	switch (idxField)
4344	{
4345	case VMX_VMCS_GUEST_RIP:
4346	case VMX_VMCS_GUEST_RSP:
4347	case VMX_VMCS_GUEST_SYSENTER_EIP:
4348	case VMX_VMCS_GUEST_SYSENTER_ESP:
4349	case VMX_VMCS_GUEST_GDTR_BASE:
4350	case VMX_VMCS_GUEST_IDTR_BASE:
4351	case VMX_VMCS_GUEST_CS_BASE:
4352	case VMX_VMCS_GUEST_DS_BASE:
4353	case VMX_VMCS_GUEST_ES_BASE:
4354	case VMX_VMCS_GUEST_FS_BASE:
4355	case VMX_VMCS_GUEST_GS_BASE:
4356	case VMX_VMCS_GUEST_SS_BASE:
4357	case VMX_VMCS_GUEST_LDTR_BASE:
4358	case VMX_VMCS_GUEST_TR_BASE:
4359	case VMX_VMCS_GUEST_CR3:
4360	return true;
4361	}
4362	return false;
4363	}
4364
4365	static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4366	{
4367	switch (idxField)
4368	{
4369	/* Read-only fields. */
4370	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4371	return true;
4372	}
4373	/* Remaining readable fields should also be writable. */
4374	return hmR0VmxIsValidWriteFieldInCache(idxField);
4375	}
4376	# endif /* VBOX_STRICT */
4377
4378
4379	/**
4380	* Executes the specified handler in 64-bit mode.
4381	*
4382	* @returns VBox status code (no informational status codes).
4383	* @param pVCpu The cross context virtual CPU structure.
4384	* @param enmOp The operation to perform.
4385	* @param cParams Number of parameters.
4386	* @param paParam Array of 32-bit parameters.
4387	*/
4388	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4389	{
4390	AssertPtr(pVCpu);
4391	PVM pVM = pVCpu->CTX_SUFF(pVM);
4392	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4393	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4394	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4395	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4396
4397	#ifdef VBOX_STRICT
4398	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4399	Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4400
4401	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4402	Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4403	#endif
4404
4405	/* Disable interrupts. */
4406	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4407
4408	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4409	RTCPUID idHostCpu = RTMpCpuId();
4410	CPUMR0SetLApic(pVCpu, idHostCpu);
4411	#endif
4412
4413	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4414
4415	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4416	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4417
4418	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4419	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4420	hmR0VmxClearVmcs(pVmcsInfo);
4421
4422	/* Leave VMX root mode and disable VMX. */
4423	VMXDisable();
4424	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4425
4426	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4427	CPUMSetHyperEIP(pVCpu, enmOp);
4428	for (int i = (int)cParams - 1; i >= 0; i--)
4429	CPUMPushHyper(pVCpu, paParam[i]);
4430
4431	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4432
4433	/* Call the switcher. */
4434	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4435	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4436
4437	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4438	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4439
4440	/* Re-enter VMX root mode. */
4441	int rc2 = VMXEnable(HCPhysCpuPage);
4442	if (RT_FAILURE(rc2))
4443	{
4444	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4445	ASMSetFlags(fOldEFlags);
4446	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4447	return rc2;
4448	}
4449
4450	/* Restore the VMCS as the current VMCS. */
4451	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4452	AssertRC(rc2);
4453	Assert(!(ASMGetFlags() & X86_EFL_IF));
4454	ASMSetFlags(fOldEFlags);
4455	return rc;
4456	}
4457
4458
4459	/**
4460	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4461	* supporting 64-bit guests.
4462	*
4463	* @returns VBox status code.
4464	* @param fResume Whether to VMLAUNCH or VMRESUME.
4465	* @param pCtx Pointer to the guest-CPU context.
4466	* @param pCache Pointer to the VMCS batch cache.
4467	* @param pVM The cross context VM structure.
4468	* @param pVCpu The cross context virtual CPU structure.
4469	*/
4470	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4471	{
4472	NOREF(fResume);
4473
4474	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4475	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4476	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4477
4478	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4479	pCache->uPos = 1;
4480	pCache->interPD = PGMGetInterPaeCR3(pVM);
4481	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4482	#endif
4483
4484	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4485	pCache->TestIn.HCPhysCpuPage = 0;
4486	pCache->TestIn.HCPhysVmcs = 0;
4487	pCache->TestIn.pCache = 0;
4488	pCache->TestOut.HCPhysVmcs = 0;
4489	pCache->TestOut.pCache = 0;
4490	pCache->TestOut.pCtx = 0;
4491	pCache->TestOut.eflags = 0;
4492	#else
4493	NOREF(pCache);
4494	#endif
4495
4496	uint32_t aParam[10];
4497	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4498	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4499	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4500	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4501	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4502	aParam[5] = 0;
4503	aParam[6] = VM_RC_ADDR(pVM, pVM);
4504	aParam[7] = 0;
4505	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4506	aParam[9] = 0;
4507
4508	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4509	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4510	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4511	#endif
4512	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4513
4514	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4515	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4516	Assert(pCtx->dr[4] == 10);
4517	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4518	#endif
4519
4520	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4521	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4522	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4523	pVmcsInfo->HCPhysVmcs));
4524	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4525	pCache->TestOut.HCPhysVmcs));
4526	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4527	pCache->TestOut.pCache));
4528	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4529	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4530	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4531	pCache->TestOut.pCtx));
4532	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4533	#endif
4534	NOREF(pCtx);
4535	return rc;
4536	}
4537	#endif
4538
4539
4540	/**
4541	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4542	* the VMCS.
4543	*
4544	* @returns VBox status code.
4545	*/
4546	static int hmR0VmxExportHostControlRegs(void)
4547	{
4548	RTCCUINTREG uReg = ASMGetCR0();
4549	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4550	AssertRCReturn(rc, rc);
4551
4552	uReg = ASMGetCR3();
4553	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4554	AssertRCReturn(rc, rc);
4555
4556	uReg = ASMGetCR4();
4557	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4558	AssertRCReturn(rc, rc);
4559	return rc;
4560	}
4561
4562
4563	/**
4564	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4565	* the host-state area in the VMCS.
4566	*
4567	* @returns VBox status code.
4568	* @param pVCpu The cross context virtual CPU structure.
4569	*/
4570	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4571	{
4572	#if HC_ARCH_BITS == 64
4573	/**
4574	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4575	* requirements. See hmR0VmxExportHostSegmentRegs().
4576	*/
4577	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4578	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4579	{ \
4580	bool fValidSelector = true; \
4581	if ((selValue) & X86_SEL_LDT) \
4582	{ \
4583	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4584	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4585	} \
4586	if (fValidSelector) \
4587	{ \
4588	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4589	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4590	} \
4591	(selValue) = 0; \
4592	}
4593
4594	/*
4595	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4596	* will be messed up. We should -not- save the messed up state without restoring
4597	* the original host-state, see @bugref{7240}.
4598	*
4599	* This apparently can happen (most likely the FPU changes), deal with it rather than
4600	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4601	*/
4602	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4603	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4604	{
4605	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4606	pVCpu->idCpu));
4607	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4608	}
4609	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4610	#else
4611	RT_NOREF(pVCpu);
4612	#endif
4613
4614	/*
4615	* Host DS, ES, FS and GS segment registers.
4616	*/
4617	#if HC_ARCH_BITS == 64
4618	RTSEL uSelDS = ASMGetDS();
4619	RTSEL uSelES = ASMGetES();
4620	RTSEL uSelFS = ASMGetFS();
4621	RTSEL uSelGS = ASMGetGS();
4622	#else
4623	RTSEL uSelDS = 0;
4624	RTSEL uSelES = 0;
4625	RTSEL uSelFS = 0;
4626	RTSEL uSelGS = 0;
4627	#endif
4628
4629	/*
4630	* Host CS and SS segment registers.
4631	*/
4632	RTSEL uSelCS = ASMGetCS();
4633	RTSEL uSelSS = ASMGetSS();
4634
4635	/*
4636	* Host TR segment register.
4637	*/
4638	RTSEL uSelTR = ASMGetTR();
4639
4640	#if HC_ARCH_BITS == 64
4641	/*
4642	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4643	* gain VM-entry and restore them before we get preempted.
4644	*
4645	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4646	*/
4647	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4648	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4649	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4650	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4651	# undef VMXLOCAL_ADJUST_HOST_SEG
4652	#endif
4653
4654	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4655	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4656	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4657	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4658	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4659	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4660	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4661	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4662	Assert(uSelCS);
4663	Assert(uSelTR);
4664
4665	/* Write these host selector fields into the host-state area in the VMCS. */
4666	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4667	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4668	#if HC_ARCH_BITS == 64
4669	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4670	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4671	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4672	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4673	#else
4674	NOREF(uSelDS);
4675	NOREF(uSelES);
4676	NOREF(uSelFS);
4677	NOREF(uSelGS);
4678	#endif
4679	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4680	AssertRCReturn(rc, rc);
4681
4682	/*
4683	* Host GDTR and IDTR.
4684	*/
4685	RTGDTR Gdtr;
4686	RTIDTR Idtr;
4687	RT_ZERO(Gdtr);
4688	RT_ZERO(Idtr);
4689	ASMGetGDTR(&Gdtr);
4690	ASMGetIDTR(&Idtr);
4691	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4692	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4693	AssertRCReturn(rc, rc);
4694
4695	#if HC_ARCH_BITS == 64
4696	/*
4697	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4698	* them to the maximum limit (0xffff) on every VM-exit.
4699	*/
4700	if (Gdtr.cbGdt != 0xffff)
4701	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4702
4703	/*
4704	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4705	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4706	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4707	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4708	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4709	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4710	* at 0xffff on hosts where we are sure it won't cause trouble.
4711	*/
4712	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4713	if (Idtr.cbIdt < 0x0fff)
4714	# else
4715	if (Idtr.cbIdt != 0xffff)
4716	# endif
4717	{
4718	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4719	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4720	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4721	}
4722	#endif
4723
4724	/*
4725	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4726	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4727	* RPL should be too in most cases.
4728	*/
4729	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4730	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4731
4732	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4733	#if HC_ARCH_BITS == 64
4734	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4735
4736	/*
4737	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4738	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4739	* restoration if the host has something else. Task switching is not supported in 64-bit
4740	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4741	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4742	*
4743	* [1] See Intel spec. 3.5 "System Descriptor Types".
4744	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4745	*/
4746	PVM pVM = pVCpu->CTX_SUFF(pVM);
4747	Assert(pDesc->System.u4Type == 11);
4748	if ( pDesc->System.u16LimitLow != 0x67
4749	\|\| pDesc->System.u4LimitHigh)
4750	{
4751	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4752	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4753	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4754	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4755	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4756	}
4757
4758	/*
4759	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4760	*/
4761	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4762	{
4763	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4764	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4765	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4766	{
4767	/* The GDT is read-only but the writable GDT is available. */
4768	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4769	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4770	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4771	AssertRCReturn(rc, rc);
4772	}
4773	}
4774	#else
4775	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4776	#endif
4777	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4778	AssertRCReturn(rc, rc);
4779
4780	/*
4781	* Host FS base and GS base.
4782	*/
4783	#if HC_ARCH_BITS == 64
4784	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4785	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4786	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4787	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4788	AssertRCReturn(rc, rc);
4789
4790	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4791	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4792	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4793	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4794	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4795	#endif
4796	return VINF_SUCCESS;
4797	}
4798
4799
4800	/**
4801	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4802	* host-state area of the VMCS.
4803	*
4804	* These MSRs will be automatically restored on the host after every successful
4805	* VM-exit.
4806	*
4807	* @returns VBox status code.
4808	* @param pVCpu The cross context virtual CPU structure.
4809	*
4810	* @remarks No-long-jump zone!!!
4811	*/
4812	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4813	{
4814	AssertPtr(pVCpu);
4815
4816	/*
4817	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4818	* rather than swapping them on every VM-entry.
4819	*/
4820	hmR0VmxLazySaveHostMsrs(pVCpu);
4821
4822	/*
4823	* Host Sysenter MSRs.
4824	*/
4825	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4826	#if HC_ARCH_BITS == 32
4827	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4828	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4829	#else
4830	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4831	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4832	#endif
4833	AssertRCReturn(rc, rc);
4834
4835	/*
4836	* Host EFER MSR.
4837	*
4838	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4839	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4840	*/
4841	PVM pVM = pVCpu->CTX_SUFF(pVM);
4842	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4843	{
4844	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4845	AssertRCReturn(rc, rc);
4846	}
4847
4848	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4849	* hmR0VmxExportGuestEntryExitCtls(). */
4850
4851	return VINF_SUCCESS;
4852	}
4853
4854
4855	/**
4856	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4857	*
4858	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4859	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4860	*
4861	* @returns true if we need to load guest EFER, false otherwise.
4862	* @param pVCpu The cross context virtual CPU structure.
4863	*
4864	* @remarks Requires EFER, CR4.
4865	* @remarks No-long-jump zone!!!
4866	*/
4867	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4868	{
4869	#ifdef HMVMX_ALWAYS_SWAP_EFER
4870	RT_NOREF(pVCpu);
4871	return true;
4872	#else
4873	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4874	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4875	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4876	if (CPUMIsGuestInLongModeEx(pCtx))
4877	return false;
4878	#endif
4879
4880	PVM pVM = pVCpu->CTX_SUFF(pVM);
4881	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4882	uint64_t const u64GuestEfer = pCtx->msrEFER;
4883
4884	/*
4885	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4886	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4887	*/
4888	if ( CPUMIsGuestInLongModeEx(pCtx)
4889	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4890	return true;
4891
4892	/*
4893	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4894	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4895	*
4896	* See Intel spec. 4.5 "IA-32e Paging".
4897	* See Intel spec. 4.1.1 "Three Paging Modes".
4898	*
4899	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4900	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4901	*/
4902	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4903	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4904	if ( (pCtx->cr4 & X86_CR4_PAE)
4905	&& (pCtx->cr0 & X86_CR0_PG)
4906	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4907	{
4908	/* Assert that host is NX capable. */
4909	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4910	return true;
4911	}
4912
4913	return false;
4914	#endif
4915	}
4916
4917	/**
4918	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4919	* VMCS.
4920	*
4921	* This is typically required when the guest changes paging mode.
4922	*
4923	* @returns VBox status code.
4924	* @param pVCpu The cross context virtual CPU structure.
4925	* @param pVmxTransient The VMX-transient structure.
4926	*
4927	* @remarks Requires EFER.
4928	* @remarks No-long-jump zone!!!
4929	*/
4930	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4931	{
4932	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4933	{
4934	PVM pVM = pVCpu->CTX_SUFF(pVM);
4935	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4936
4937	/*
4938	* VM-entry controls.
4939	*/
4940	{
4941	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4942	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4943
4944	/*
4945	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4946	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4947	*
4948	* For nested-guests, this is a mandatory VM-entry control. It's also
4949	* required because we do not want to leak host bits to the nested-guest.
4950	*/
4951	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4952
4953	/*
4954	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4955	*
4956	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4957	* required to get the nested-guest working with hardware-assisted VMX execution.
4958	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4959	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4960	* here rather than while merging the guest VMCS controls.
4961	*/
4962	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4963	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4964	else
4965	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4966
4967	/*
4968	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4969	*
4970	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4971	* regardless of whether the nested-guest VMCS specifies it because we are free to
4972	* load whatever MSRs we require and we do not need to modify the guest visible copy
4973	* of the VM-entry MSR load area.
4974	*/
4975	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4976	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4977	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4978	else
4979	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4980
4981	/*
4982	* The following should -not- be set (since we're not in SMM mode):
4983	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4984	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4985	*/
4986
4987	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4988	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4989
4990	if ((fVal & fZap) == fVal)
4991	{ /* likely */ }
4992	else
4993	{
4994	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4995	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4996	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4997	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4998	}
4999
5000	/* Commit it to the VMCS. */
5001	if (pVmcsInfo->u32EntryCtls != fVal)
5002	{
5003	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
5004	AssertRCReturn(rc, rc);
5005	pVmcsInfo->u32EntryCtls = fVal;
5006	}
5007	}
5008
5009	/*
5010	* VM-exit controls.
5011	*/
5012	{
5013	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
5014	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
5015
5016	/*
5017	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
5018	* supported the 1-setting of this bit.
5019	*
5020	* For nested-guests, we set the "save debug controls" as the converse
5021	* "load debug controls" is mandatory for nested-guests anyway.
5022	*/
5023	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
5024
5025	/*
5026	* Set the host long mode active (EFER.LMA) bit (which Intel calls
5027	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
5028	* host EFER.LMA and EFER.LME bit to this value. See assertion in
5029	* hmR0VmxExportHostMsrs().
5030	*
5031	* For nested-guests, we always set this bit as we do not support 32-bit
5032	* hosts.
5033	*/
5034	#if HC_ARCH_BITS == 64
5035	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5036	#else
5037	Assert(!pVmxTransient->fIsNestedGuest);
5038	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5039	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5040	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5041	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5042	{
5043	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5044	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5045	}
5046	else
5047	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5048	#endif
5049
5050	/*
5051	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
5052	*
5053	* For nested-guests, we should use the "save IA32_EFER" control if we also
5054	* used the "load IA32_EFER" control while exporting VM-entry controls.
5055	*/
5056	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5057	&& hmR0VmxShouldSwapEferMsr(pVCpu))
5058	{
5059	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
5060	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
5061	}
5062
5063	/*
5064	* Enable saving of the VMX-preemption timer value on VM-exit.
5065	* For nested-guests, currently not exposed/used.
5066	*/
5067	if ( pVM->hm.s.vmx.fUsePreemptTimer
5068	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5069	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5070
5071	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5072	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5073
5074	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5075	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
5076	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5077
5078	if ((fVal & fZap) == fVal)
5079	{ /* likely */ }
5080	else
5081	{
5082	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5083	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5084	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5085	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5086	}
5087
5088	/* Commit it to the VMCS. */
5089	if (pVmcsInfo->u32ExitCtls != fVal)
5090	{
5091	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5092	AssertRCReturn(rc, rc);
5093	pVmcsInfo->u32ExitCtls = fVal;
5094	}
5095	}
5096
5097	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5098	}
5099	return VINF_SUCCESS;
5100	}
5101
5102
5103	/**
5104	* Sets the TPR threshold in the VMCS.
5105	*
5106	* @returns VBox status code.
5107	* @param pVCpu The cross context virtual CPU structure.
5108	* @param pVmcsInfo The VMCS info. object.
5109	* @param u32TprThreshold The TPR threshold (task-priority class only).
5110	*/
5111	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5112	{
5113	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5114	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5115	RT_NOREF2(pVCpu, pVmcsInfo);
5116	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5117	}
5118
5119
5120	/**
5121	* Exports the guest APIC TPR state into the VMCS.
5122	*
5123	* @returns VBox status code.
5124	* @param pVCpu The cross context virtual CPU structure.
5125	* @param pVmxTransient The VMX-transient structure.
5126	*
5127	* @remarks No-long-jump zone!!!
5128	*/
5129	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5130	{
5131	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5132	{
5133	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5134
5135	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5136	if (!pVmxTransient->fIsNestedGuest)
5137	{
5138	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5139	&& APICIsEnabled(pVCpu))
5140	{
5141	/*
5142	* Setup TPR shadowing.
5143	*/
5144	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5145	{
5146	bool fPendingIntr = false;
5147	uint8_t u8Tpr = 0;
5148	uint8_t u8PendingIntr = 0;
5149	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5150	AssertRCReturn(rc, rc);
5151
5152	/*
5153	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5154	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5155	* priority of the pending interrupt so we can deliver the interrupt. If there
5156	* are no interrupts pending, set threshold to 0 to not cause any
5157	* TPR-below-threshold VM-exits.
5158	*/
5159	Assert(pVmcsInfo->pbVirtApic);
5160	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5161	uint32_t u32TprThreshold = 0;
5162	if (fPendingIntr)
5163	{
5164	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5165	(which is the Task-Priority Class). */
5166	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5167	const uint8_t u8TprPriority = u8Tpr >> 4;
5168	if (u8PendingPriority <= u8TprPriority)
5169	u32TprThreshold = u8PendingPriority;
5170	}
5171
5172	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5173	AssertRCReturn(rc, rc);
5174	}
5175	}
5176	}
5177	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5178	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5179	}
5180	return VINF_SUCCESS;
5181	}
5182
5183
5184	/**
5185	* Gets the guest interruptibility-state.
5186	*
5187	* @returns Guest's interruptibility-state.
5188	* @param pVCpu The cross context virtual CPU structure.
5189	* @param pVmxTransient The VMX-transient structure.
5190	*
5191	* @remarks No-long-jump zone!!!
5192	*/
5193	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5194	{
5195	/*
5196	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5197	*/
5198	uint32_t fIntrState = 0;
5199	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5200	{
5201	/* If inhibition is active, RIP and RFLAGS should've been updated
5202	(i.e. read previously from the VMCS or from ring-3). */
5203	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5204	#ifdef VBOX_STRICT
5205	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5206	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5207	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5208	#endif
5209	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5210	{
5211	if (pCtx->eflags.Bits.u1IF)
5212	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5213	else
5214	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5215	}
5216	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5217	{
5218	/*
5219	* We can clear the inhibit force flag as even if we go back to the recompiler
5220	* without executing guest code in VT-x, the flag's condition to be cleared is
5221	* met and thus the cleared state is correct.
5222	*/
5223	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5224	}
5225	}
5226
5227	/*
5228	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5229	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5230	* setting this would block host-NMIs and IRET will not clear the blocking.
5231	*
5232	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5233	*
5234	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5235	*/
5236	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5237	&& CPUMIsGuestNmiBlocking(pVCpu))
5238	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5239
5240	return fIntrState;
5241	}
5242
5243
5244	/**
5245	* Exports the exception intercepts required for guest execution in the VMCS.
5246	*
5247	* @returns VBox status code.
5248	* @param pVCpu The cross context virtual CPU structure.
5249	* @param pVmxTransient The VMX-transient structure.
5250	*
5251	* @remarks No-long-jump zone!!!
5252	*/
5253	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5254	{
5255	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5256	{
5257	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5258	if ( !pVmxTransient->fIsNestedGuest
5259	&& pVCpu->hm.s.fGIMTrapXcptUD)
5260	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5261	else
5262	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5263
5264	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5265	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5266	}
5267	return VINF_SUCCESS;
5268	}
5269
5270
5271	/**
5272	* Exports the guest's RIP into the guest-state area in the VMCS.
5273	*
5274	* @returns VBox status code.
5275	* @param pVCpu The cross context virtual CPU structure.
5276	*
5277	* @remarks No-long-jump zone!!!
5278	*/
5279	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5280	{
5281	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5282	{
5283	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5284
5285	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5286	AssertRCReturn(rc, rc);
5287
5288	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5289	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5290	}
5291	return VINF_SUCCESS;
5292	}
5293
5294
5295	/**
5296	* Exports the guest's RSP into the guest-state area in the VMCS.
5297	*
5298	* @returns VBox status code.
5299	* @param pVCpu The cross context virtual CPU structure.
5300	*
5301	* @remarks No-long-jump zone!!!
5302	*/
5303	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5304	{
5305	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5306	{
5307	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5308
5309	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5310	AssertRCReturn(rc, rc);
5311
5312	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5313	}
5314	return VINF_SUCCESS;
5315	}
5316
5317
5318	/**
5319	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5320	*
5321	* @returns VBox status code.
5322	* @param pVCpu The cross context virtual CPU structure.
5323	* @param pVmxTransient The VMX-transient structure.
5324	*
5325	* @remarks No-long-jump zone!!!
5326	*/
5327	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5328	{
5329	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5330	{
5331	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5332
5333	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5334	Let us assert it as such and use 32-bit VMWRITE. */
5335	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5336	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5337	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5338	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5339
5340	/*
5341	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5342	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5343	* can run the real-mode guest code under Virtual 8086 mode.
5344	*/
5345	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5346	if (pVmcsInfo->RealMode.fRealOnV86Active)
5347	{
5348	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5349	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5350	Assert(!pVmxTransient->fIsNestedGuest);
5351	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5352	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5353	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5354	}
5355
5356	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5357	AssertRCReturn(rc, rc);
5358
5359	/*
5360	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5361	*
5362	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5363	* through the hypervisor debugger using EFLAGS.TF.
5364	*/
5365	if ( !pVmxTransient->fIsNestedGuest
5366	&& !pVCpu->hm.s.fSingleInstruction
5367	&& fEFlags.Bits.u1TF)
5368	{
5369	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5370	* premature trips to ring-3 esp since IEM does not yet handle it. */
5371	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5372	AssertRCReturn(rc, rc);
5373	}
5374	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5375	* nested-guest VMCS. */
5376
5377	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5378	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5379	}
5380	return VINF_SUCCESS;
5381	}
5382
5383
5384	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5385	/**
5386	* Copies the nested-guest VMCS to the shadow VMCS.
5387	*
5388	* @returns VBox status code.
5389	* @param pVCpu The cross context virtual CPU structure.
5390	* @param pVmcsInfo The VMCS info. object.
5391	*
5392	* @remarks No-long-jump zone!!!
5393	*/
5394	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5395	{
5396	PVM pVM = pVCpu->CTX_SUFF(pVM);
5397	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5398
5399	/*
5400	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5401	* current VMCS, as we may try saving guest lazy MSRs.
5402	*
5403	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5404	* calling the import VMCS code which is currently performing the guest MSR reads
5405	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5406	* and the rest of the VMX leave session machinery.
5407	*/
5408	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5409
5410	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5411	if (RT_SUCCESS(rc))
5412	{
5413	/*
5414	* Copy all guest read/write VMCS fields.
5415	*
5416	* We don't check for VMWRITE failures here for performance reasons and
5417	* because they are not expected to fail, barring irrecoverable conditions
5418	* like hardware errors.
5419	*/
5420	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5421	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5422	{
5423	uint64_t u64Val;
5424	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5425	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5426	VMXWriteVmcs64(uVmcsField, u64Val);
5427	}
5428
5429	/*
5430	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5431	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5432	*/
5433	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5434	{
5435	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5436	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5437	{
5438	uint64_t u64Val;
5439	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5440	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5441	VMXWriteVmcs64(uVmcsField, u64Val);
5442	}
5443	}
5444
5445	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5446	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5447	}
5448
5449	ASMSetFlags(fEFlags);
5450	return rc;
5451	}
5452
5453
5454	/**
5455	* Copies the shadow VMCS to the nested-guest VMCS.
5456	*
5457	* @returns VBox status code.
5458	* @param pVCpu The cross context virtual CPU structure.
5459	* @param pVmcsInfo The VMCS info. object.
5460	*
5461	* @remarks Called with interrupts disabled.
5462	*/
5463	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5464	{
5465	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5466	PVM pVM = pVCpu->CTX_SUFF(pVM);
5467	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5468
5469	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5470	if (RT_SUCCESS(rc))
5471	{
5472	/*
5473	* Copy guest read/write fields from the shadow VMCS.
5474	* Guest read-only fields cannot be modified, so no need to copy them.
5475	*
5476	* We don't check for VMREAD failures here for performance reasons and
5477	* because they are not expected to fail, barring irrecoverable conditions
5478	* like hardware errors.
5479	*/
5480	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5481	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5482	{
5483	uint64_t u64Val;
5484	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5485	VMXReadVmcs64(uVmcsField, &u64Val);
5486	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5487	}
5488
5489	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5490	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5491	}
5492	return rc;
5493	}
5494
5495
5496	/**
5497	* Enables VMCS shadowing for the given VMCS info. object.
5498	*
5499	* @param pVCpu The cross context virtual CPU structure.
5500	* @param pVmcsInfo The VMCS info. object.
5501	*
5502	* @remarks No-long-jump zone!!!
5503	*/
5504	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5505	{
5506	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5507
5508	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5509	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5510	{
5511	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5512	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5513	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5514	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5515	AssertRC(rc);
5516
5517	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5518	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5519	Log4Func(("Enabled\n"));
5520	}
5521	}
5522
5523
5524	/**
5525	* Disables VMCS shadowing for the given VMCS info. object.
5526	*
5527	* @param pVCpu The cross context virtual CPU structure.
5528	* @param pVmcsInfo The VMCS info. object.
5529	*
5530	* @remarks No-long-jump zone!!!
5531	*/
5532	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5533	{
5534	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5535
5536	/*
5537	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5538	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5539	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5540	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5541	*
5542	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5543	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5544	*/
5545	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5546	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5547	{
5548	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5549	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5550	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5551	AssertRC(rc);
5552
5553	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5554	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5555	Log4Func(("Disabled\n"));
5556	}
5557	}
5558	#endif
5559
5560
5561	/**
5562	* Exports the guest hardware-virtualization state.
5563	*
5564	* @returns VBox status code.
5565	* @param pVCpu The cross context virtual CPU structure.
5566	* @param pVmxTransient The VMX-transient structure.
5567	*
5568	* @remarks No-long-jump zone!!!
5569	*/
5570	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5571	{
5572	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5573	{
5574	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5575	/*
5576	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5577	* VMCS shadowing.
5578	*/
5579	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5580	{
5581	/*
5582	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5583	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5584	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5585	*
5586	* We check for VMX root mode here in case the guest executes VMXOFF without
5587	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5588	* not clear the current VMCS pointer.
5589	*/
5590	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5591	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5592	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5593	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5594	{
5595	/* Paranoia. */
5596	Assert(!pVmxTransient->fIsNestedGuest);
5597
5598	/*
5599	* For performance reasons, also check if the guest hypervisor's current VMCS
5600	* was newly loaded or modified before copying it to the shadow VMCS.
5601	*/
5602	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5603	{
5604	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5605	AssertRCReturn(rc, rc);
5606	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5607	}
5608	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5609	}
5610	else
5611	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5612	}
5613	#else
5614	NOREF(pVmxTransient);
5615	#endif
5616	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5617	}
5618	return VINF_SUCCESS;
5619	}
5620
5621
5622	/**
5623	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5624	*
5625	* The guest FPU state is always pre-loaded hence we don't need to bother about
5626	* sharing FPU related CR0 bits between the guest and host.
5627	*
5628	* @returns VBox status code.
5629	* @param pVCpu The cross context virtual CPU structure.
5630	* @param pVmxTransient The VMX-transient structure.
5631	*
5632	* @remarks No-long-jump zone!!!
5633	*/
5634	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5635	{
5636	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5637	{
5638	PVM pVM = pVCpu->CTX_SUFF(pVM);
5639	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5640
5641	/*
5642	* Figure out fixed CR0 bits in VMX operation.
5643	*/
5644	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5645	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5646	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5647	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5648	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5649	else
5650	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5651
5652	if (!pVmxTransient->fIsNestedGuest)
5653	{
5654	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5655	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5656	uint64_t const u64ShadowCr0 = u64GuestCr0;
5657	Assert(!RT_HI_U32(u64GuestCr0));
5658
5659	/*
5660	* Setup VT-x's view of the guest CR0.
5661	*/
5662	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5663	if (pVM->hm.s.fNestedPaging)
5664	{
5665	if (CPUMIsGuestPagingEnabled(pVCpu))
5666	{
5667	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5668	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5669	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5670	}
5671	else
5672	{
5673	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5674	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5675	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5676	}
5677
5678	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5679	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5680	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5681	}
5682	else
5683	{
5684	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5685	u64GuestCr0 \|= X86_CR0_WP;
5686	}
5687
5688	/*
5689	* Guest FPU bits.
5690	*
5691	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5692	* using CR0.TS.
5693	*
5694	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5695	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5696	*/
5697	u64GuestCr0 \|= X86_CR0_NE;
5698
5699	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5700	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5701
5702	/*
5703	* Update exception intercepts.
5704	*/
5705	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5706	if (pVmcsInfo->RealMode.fRealOnV86Active)
5707	{
5708	Assert(PDMVmmDevHeapIsEnabled(pVM));
5709	Assert(pVM->hm.s.vmx.pRealModeTSS);
5710	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5711	}
5712	else
5713	{
5714	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5715	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5716	if (fInterceptMF)
5717	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5718	}
5719
5720	/* Additional intercepts for debugging, define these yourself explicitly. */
5721	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5722	uXcptBitmap \|= 0
5723	\| RT_BIT(X86_XCPT_BP)
5724	\| RT_BIT(X86_XCPT_DE)
5725	\| RT_BIT(X86_XCPT_NM)
5726	\| RT_BIT(X86_XCPT_TS)
5727	\| RT_BIT(X86_XCPT_UD)
5728	\| RT_BIT(X86_XCPT_NP)
5729	\| RT_BIT(X86_XCPT_SS)
5730	\| RT_BIT(X86_XCPT_GP)
5731	\| RT_BIT(X86_XCPT_PF)
5732	\| RT_BIT(X86_XCPT_MF)
5733	;
5734	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5735	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5736	#endif
5737	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5738	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5739	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5740
5741	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5742	u64GuestCr0 \|= fSetCr0;
5743	u64GuestCr0 &= fZapCr0;
5744	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5745
5746	/* Commit the CR0 and related fields to the guest VMCS. */
5747	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5748	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5749	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5750	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5751	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5752	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5753	AssertRCReturn(rc, rc);
5754
5755	/* Update our caches. */
5756	pVmcsInfo->u32ProcCtls = uProcCtls;
5757	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5758
5759	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5760	}
5761	else
5762	{
5763	/*
5764	* With nested-guests, we may have extended the guest/host mask here since we
5765	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5766	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5767	* originally supplied. We must copy those bits from the nested-guest CR0 into
5768	* the nested-guest CR0 read-shadow.
5769	*/
5770	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5771	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5772	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5773	Assert(!RT_HI_U32(u64GuestCr0));
5774	Assert(u64GuestCr0 & X86_CR0_NE);
5775
5776	/*
5777	* Apply the hardware specified fixed CR0 bits and enable caching.
5778	* Note! We could be altering our VMX emulation's fixed bits. We thus
5779	* need to re-apply them while importing CR0.
5780	*/
5781	u64GuestCr0 \|= fSetCr0;
5782	u64GuestCr0 &= fZapCr0;
5783	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5784
5785	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5786	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5787	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5788	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5789	AssertRCReturn(rc, rc);
5790
5791	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5792	}
5793
5794	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5795	}
5796
5797	return VINF_SUCCESS;
5798	}
5799
5800
5801	/**
5802	* Exports the guest control registers (CR3, CR4) into the guest-state area
5803	* in the VMCS.
5804	*
5805	* @returns VBox strict status code.
5806	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5807	* without unrestricted guest access and the VMMDev is not presently
5808	* mapped (e.g. EFI32).
5809	*
5810	* @param pVCpu The cross context virtual CPU structure.
5811	* @param pVmxTransient The VMX-transient structure.
5812	*
5813	* @remarks No-long-jump zone!!!
5814	*/
5815	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5816	{
5817	int rc = VINF_SUCCESS;
5818	PVM pVM = pVCpu->CTX_SUFF(pVM);
5819
5820	/*
5821	* Guest CR2.
5822	* It's always loaded in the assembler code. Nothing to do here.
5823	*/
5824
5825	/*
5826	* Guest CR3.
5827	*/
5828	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5829	{
5830	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5831
5832	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5833	if (pVM->hm.s.fNestedPaging)
5834	{
5835	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5836	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5837
5838	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5839	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5840	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5841	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5842
5843	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5844	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5845	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5846
5847	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5848	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5849	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5850	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5851	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5852	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5853	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5854
5855	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5856	AssertRCReturn(rc, rc);
5857
5858	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5859	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5860	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5861	{
5862	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5863	if (CPUMIsGuestInPAEModeEx(pCtx))
5864	{
5865	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5866	AssertRCReturn(rc, rc);
5867	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5868	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5869	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5870	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5871	AssertRCReturn(rc, rc);
5872	}
5873
5874	/*
5875	* The guest's view of its CR3 is unblemished with nested paging when the
5876	* guest is using paging or we have unrestricted guest execution to handle
5877	* the guest when it's not using paging.
5878	*/
5879	GCPhysGuestCr3 = pCtx->cr3;
5880	}
5881	else
5882	{
5883	/*
5884	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5885	* thinks it accesses physical memory directly, we use our identity-mapped
5886	* page table to map guest-linear to guest-physical addresses. EPT takes care
5887	* of translating it to host-physical addresses.
5888	*/
5889	RTGCPHYS GCPhys;
5890	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5891
5892	/* We obtain it here every time as the guest could have relocated this PCI region. */
5893	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5894	if (RT_SUCCESS(rc))
5895	{ /* likely */ }
5896	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5897	{
5898	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5899	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5900	}
5901	else
5902	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5903
5904	GCPhysGuestCr3 = GCPhys;
5905	}
5906
5907	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5908	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5909	AssertRCReturn(rc, rc);
5910	}
5911	else
5912	{
5913	/* Non-nested paging case, just use the hypervisor's CR3. */
5914	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5915
5916	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5917	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5918	AssertRCReturn(rc, rc);
5919	}
5920
5921	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5922	}
5923
5924	/*
5925	* Guest CR4.
5926	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5927	*/
5928	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5929	{
5930	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5931	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5932
5933	/*
5934	* Figure out fixed CR4 bits in VMX operation.
5935	*/
5936	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5937	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5938	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5939
5940	/*
5941	* With nested-guests, we may have extended the guest/host mask here (since we
5942	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5943	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5944	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5945	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5946	*/
5947	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5948	uint64_t u64GuestCr4 = pCtx->cr4;
5949	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5950	? pCtx->cr4
5951	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5952	Assert(!RT_HI_U32(u64GuestCr4));
5953
5954	/*
5955	* Setup VT-x's view of the guest CR4.
5956	*
5957	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5958	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5959	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5960	*
5961	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5962	*/
5963	if (pVmcsInfo->RealMode.fRealOnV86Active)
5964	{
5965	Assert(pVM->hm.s.vmx.pRealModeTSS);
5966	Assert(PDMVmmDevHeapIsEnabled(pVM));
5967	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5968	}
5969
5970	if (pVM->hm.s.fNestedPaging)
5971	{
5972	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5973	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5974	{
5975	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5976	u64GuestCr4 \|= X86_CR4_PSE;
5977	/* Our identity mapping is a 32-bit page directory. */
5978	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5979	}
5980	/* else use guest CR4.*/
5981	}
5982	else
5983	{
5984	Assert(!pVmxTransient->fIsNestedGuest);
5985
5986	/*
5987	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5988	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5989	*/
5990	switch (pVCpu->hm.s.enmShadowMode)
5991	{
5992	case PGMMODE_REAL: /* Real-mode. */
5993	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5994	case PGMMODE_32_BIT: /* 32-bit paging. */
5995	{
5996	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5997	break;
5998	}
5999
6000	case PGMMODE_PAE: /* PAE paging. */
6001	case PGMMODE_PAE_NX: /* PAE paging with NX. */
6002	{
6003	u64GuestCr4 \|= X86_CR4_PAE;
6004	break;
6005	}
6006
6007	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
6008	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
6009	#ifdef VBOX_ENABLE_64_BITS_GUESTS
6010	break;
6011	#endif
6012	default:
6013	AssertFailed();
6014	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6015	}
6016	}
6017
6018	/*
6019	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
6020	* Note! For nested-guests, we could be altering our VMX emulation's
6021	* fixed bits. We thus need to re-apply them while importing CR4.
6022	*/
6023	u64GuestCr4 \|= fSetCr4;
6024	u64GuestCr4 &= fZapCr4;
6025
6026	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
6027	/** @todo Fix to 64-bit when we drop 32-bit. */
6028	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
6029	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
6030	AssertRCReturn(rc, rc);
6031
6032	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
6033	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6034
6035	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
6036
6037	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
6038	}
6039	return rc;
6040	}
6041
6042
6043	/**
6044	* Exports the guest debug registers into the guest-state area in the VMCS.
6045	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
6046	*
6047	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
6048	*
6049	* @returns VBox status code.
6050	* @param pVCpu The cross context virtual CPU structure.
6051	* @param pVmxTransient The VMX-transient structure.
6052	*
6053	* @remarks No-long-jump zone!!!
6054	*/
6055	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6056	{
6057	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6058
6059	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
6060	* stepping. */
6061	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6062	if (pVmxTransient->fIsNestedGuest)
6063	{
6064	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
6065	AssertRCReturn(rc, rc);
6066	return VINF_SUCCESS;
6067	}
6068
6069	#ifdef VBOX_STRICT
6070	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
6071	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
6072	{
6073	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
6074	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
6075	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
6076	}
6077	#endif
6078
6079	bool fSteppingDB = false;
6080	bool fInterceptMovDRx = false;
6081	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6082	if (pVCpu->hm.s.fSingleInstruction)
6083	{
6084	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
6085	PVM pVM = pVCpu->CTX_SUFF(pVM);
6086	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
6087	{
6088	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
6089	Assert(fSteppingDB == false);
6090	}
6091	else
6092	{
6093	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
6094	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
6095	pVCpu->hm.s.fClearTrapFlag = true;
6096	fSteppingDB = true;
6097	}
6098	}
6099
6100	uint32_t u32GuestDr7;
6101	if ( fSteppingDB
6102	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
6103	{
6104	/*
6105	* Use the combined guest and host DRx values found in the hypervisor register set
6106	* because the hypervisor debugger has breakpoints active or someone is single stepping
6107	* on the host side without a monitor trap flag.
6108	*
6109	* Note! DBGF expects a clean DR6 state before executing guest code.
6110	*/
6111	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6112	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6113	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
6114	{
6115	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6116	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
6117	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
6118	}
6119	else
6120	#endif
6121	if (!CPUMIsHyperDebugStateActive(pVCpu))
6122	{
6123	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6124	Assert(CPUMIsHyperDebugStateActive(pVCpu));
6125	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6126	}
6127
6128	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
6129	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
6130	pVCpu->hm.s.fUsingHyperDR7 = true;
6131	fInterceptMovDRx = true;
6132	}
6133	else
6134	{
6135	/*
6136	* If the guest has enabled debug registers, we need to load them prior to
6137	* executing guest code so they'll trigger at the right time.
6138	*/
6139	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6140	{
6141	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6142	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6143	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
6144	{
6145	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6146	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
6147	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
6148	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6149	}
6150	else
6151	#endif
6152	if (!CPUMIsGuestDebugStateActive(pVCpu))
6153	{
6154	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6155	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6156	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6157	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6158	}
6159	Assert(!fInterceptMovDRx);
6160	}
6161	/*
6162	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6163	* must intercept #DB in order to maintain a correct DR6 guest value, and
6164	* because we need to intercept it to prevent nested #DBs from hanging the
6165	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6166	*/
6167	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6168	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
6169	&& !CPUMIsGuestDebugStateActive(pVCpu))
6170	#else
6171	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6172	#endif
6173	{
6174	fInterceptMovDRx = true;
6175	}
6176
6177	/* Update DR7 with the actual guest value. */
6178	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6179	pVCpu->hm.s.fUsingHyperDR7 = false;
6180	}
6181
6182	if (fInterceptMovDRx)
6183	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6184	else
6185	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6186
6187	/*
6188	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6189	* monitor-trap flag and update our cache.
6190	*/
6191	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6192	{
6193	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6194	AssertRCReturn(rc2, rc2);
6195	pVmcsInfo->u32ProcCtls = uProcCtls;
6196	}
6197
6198	/*
6199	* Update guest DR7.
6200	*/
6201	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6202	AssertRCReturn(rc, rc);
6203
6204	/*
6205	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6206	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6207	*
6208	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6209	*/
6210	if (fSteppingDB)
6211	{
6212	Assert(pVCpu->hm.s.fSingleInstruction);
6213	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6214
6215	uint32_t fIntrState = 0;
6216	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6217	AssertRCReturn(rc, rc);
6218
6219	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6220	{
6221	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6222	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6223	AssertRCReturn(rc, rc);
6224	}
6225	}
6226
6227	return VINF_SUCCESS;
6228	}
6229
6230
6231	#ifdef VBOX_STRICT
6232	/**
6233	* Strict function to validate segment registers.
6234	*
6235	* @param pVCpu The cross context virtual CPU structure.
6236	* @param pVmcsInfo The VMCS info. object.
6237	*
6238	* @remarks Will import guest CR0 on strict builds during validation of
6239	* segments.
6240	*/
6241	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6242	{
6243	/*
6244	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6245	*
6246	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6247	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6248	* unusable bit and doesn't change the guest-context value.
6249	*/
6250	PVM pVM = pVCpu->CTX_SUFF(pVM);
6251	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6252	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6253	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6254	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6255	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6256	{
6257	/* Protected mode checks */
6258	/* CS */
6259	Assert(pCtx->cs.Attr.n.u1Present);
6260	Assert(!(pCtx->cs.Attr.u & 0xf00));
6261	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6262	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6263	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6264	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6265	\|\| (pCtx->cs.Attr.n.u1Granularity));
6266	/* CS cannot be loaded with NULL in protected mode. */
6267	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6268	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6269	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6270	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6271	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6272	else
6273	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6274	/* SS */
6275	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6276	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6277	if ( !(pCtx->cr0 & X86_CR0_PE)
6278	\|\| pCtx->cs.Attr.n.u4Type == 3)
6279	{
6280	Assert(!pCtx->ss.Attr.n.u2Dpl);
6281	}
6282	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6283	{
6284	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6285	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6286	Assert(pCtx->ss.Attr.n.u1Present);
6287	Assert(!(pCtx->ss.Attr.u & 0xf00));
6288	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6289	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6290	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6291	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6292	\|\| (pCtx->ss.Attr.n.u1Granularity));
6293	}
6294	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6295	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6296	{
6297	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6298	Assert(pCtx->ds.Attr.n.u1Present);
6299	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6300	Assert(!(pCtx->ds.Attr.u & 0xf00));
6301	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6302	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6303	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6304	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6305	\|\| (pCtx->ds.Attr.n.u1Granularity));
6306	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6307	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6308	}
6309	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6310	{
6311	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6312	Assert(pCtx->es.Attr.n.u1Present);
6313	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6314	Assert(!(pCtx->es.Attr.u & 0xf00));
6315	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6316	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6317	\|\| !(pCtx->es.Attr.n.u1Granularity));
6318	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6319	\|\| (pCtx->es.Attr.n.u1Granularity));
6320	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6321	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6322	}
6323	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6324	{
6325	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6326	Assert(pCtx->fs.Attr.n.u1Present);
6327	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6328	Assert(!(pCtx->fs.Attr.u & 0xf00));
6329	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6330	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6331	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6332	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6333	\|\| (pCtx->fs.Attr.n.u1Granularity));
6334	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6335	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6336	}
6337	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6338	{
6339	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6340	Assert(pCtx->gs.Attr.n.u1Present);
6341	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6342	Assert(!(pCtx->gs.Attr.u & 0xf00));
6343	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6344	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6345	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6346	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6347	\|\| (pCtx->gs.Attr.n.u1Granularity));
6348	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6349	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6350	}
6351	/* 64-bit capable CPUs. */
6352	# if HC_ARCH_BITS == 64
6353	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6354	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6355	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6356	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6357	# endif
6358	}
6359	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6360	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6361	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6362	{
6363	/* Real and v86 mode checks. */
6364	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6365	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6366	if (pVmcsInfo->RealMode.fRealOnV86Active)
6367	{
6368	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6369	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6370	}
6371	else
6372	{
6373	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6374	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6375	}
6376
6377	/* CS */
6378	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6379	Assert(pCtx->cs.u32Limit == 0xffff);
6380	Assert(u32CSAttr == 0xf3);
6381	/* SS */
6382	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6383	Assert(pCtx->ss.u32Limit == 0xffff);
6384	Assert(u32SSAttr == 0xf3);
6385	/* DS */
6386	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6387	Assert(pCtx->ds.u32Limit == 0xffff);
6388	Assert(u32DSAttr == 0xf3);
6389	/* ES */
6390	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6391	Assert(pCtx->es.u32Limit == 0xffff);
6392	Assert(u32ESAttr == 0xf3);
6393	/* FS */
6394	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6395	Assert(pCtx->fs.u32Limit == 0xffff);
6396	Assert(u32FSAttr == 0xf3);
6397	/* GS */
6398	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6399	Assert(pCtx->gs.u32Limit == 0xffff);
6400	Assert(u32GSAttr == 0xf3);
6401	/* 64-bit capable CPUs. */
6402	# if HC_ARCH_BITS == 64
6403	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6404	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6405	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6406	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6407	# endif
6408	}
6409	}
6410	#endif /* VBOX_STRICT */
6411
6412
6413	/**
6414	* Exports a guest segment register into the guest-state area in the VMCS.
6415	*
6416	* @returns VBox status code.
6417	* @param pVCpu The cross context virtual CPU structure.
6418	* @param pVmcsInfo The VMCS info. object.
6419	* @param iSegReg The segment register number (X86_SREG_XXX).
6420	* @param pSelReg Pointer to the segment selector.
6421	*
6422	* @remarks No-long-jump zone!!!
6423	*/
6424	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6425	{
6426	Assert(iSegReg < X86_SREG_COUNT);
6427	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6428	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6429	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6430	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6431
6432	uint32_t u32Access = pSelReg->Attr.u;
6433	if (pVmcsInfo->RealMode.fRealOnV86Active)
6434	{
6435	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6436	u32Access = 0xf3;
6437	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6438	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6439	RT_NOREF_PV(pVCpu);
6440	}
6441	else
6442	{
6443	/*
6444	* The way to differentiate between whether this is really a null selector or was just
6445	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6446	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6447	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6448	* NULL selectors loaded in protected-mode have their attribute as 0.
6449	*/
6450	if (!u32Access)
6451	u32Access = X86DESCATTR_UNUSABLE;
6452	}
6453
6454	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6455	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6456	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6457
6458	/*
6459	* Commit it to the VMCS.
6460	*/
6461	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6462	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6463	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6464	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6465	AssertRCReturn(rc, rc);
6466	return rc;
6467	}
6468
6469
6470	/**
6471	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6472	* area in the VMCS.
6473	*
6474	* @returns VBox status code.
6475	* @param pVCpu The cross context virtual CPU structure.
6476	* @param pVmxTransient The VMX-transient structure.
6477	*
6478	* @remarks Will import guest CR0 on strict builds during validation of
6479	* segments.
6480	* @remarks No-long-jump zone!!!
6481	*/
6482	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6483	{
6484	int rc = VERR_INTERNAL_ERROR_5;
6485	PVM pVM = pVCpu->CTX_SUFF(pVM);
6486	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6487	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6488
6489	/*
6490	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6491	*/
6492	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6493	{
6494	#ifdef VBOX_WITH_REM
6495	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6496	{
6497	Assert(!pVmxTransient->fIsNestedGuest);
6498	Assert(pVM->hm.s.vmx.pRealModeTSS);
6499	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6500	if ( pVmcsInfo->fWasInRealMode
6501	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6502	{
6503	/*
6504	* Notify the recompiler must flush its code-cache as the guest -may-
6505	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6506	*/
6507	REMFlushTBs(pVM);
6508	Log4Func(("Switch to protected mode detected!\n"));
6509	pVmcsInfo->fWasInRealMode = false;
6510	}
6511	}
6512	#endif
6513	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6514	{
6515	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6516	if (pVmcsInfo->RealMode.fRealOnV86Active)
6517	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6518	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6519	AssertRCReturn(rc, rc);
6520	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6521	}
6522
6523	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6524	{
6525	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6526	if (pVmcsInfo->RealMode.fRealOnV86Active)
6527	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6528	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6529	AssertRCReturn(rc, rc);
6530	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6531	}
6532
6533	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6534	{
6535	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6536	if (pVmcsInfo->RealMode.fRealOnV86Active)
6537	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6538	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6539	AssertRCReturn(rc, rc);
6540	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6541	}
6542
6543	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6544	{
6545	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6546	if (pVmcsInfo->RealMode.fRealOnV86Active)
6547	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6548	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6549	AssertRCReturn(rc, rc);
6550	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6551	}
6552
6553	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6554	{
6555	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6556	if (pVmcsInfo->RealMode.fRealOnV86Active)
6557	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6558	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6559	AssertRCReturn(rc, rc);
6560	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6561	}
6562
6563	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6564	{
6565	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6566	if (pVmcsInfo->RealMode.fRealOnV86Active)
6567	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6568	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6569	AssertRCReturn(rc, rc);
6570	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6571	}
6572
6573	#ifdef VBOX_STRICT
6574	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6575	#endif
6576	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6577	pCtx->cs.Attr.u));
6578	}
6579
6580	/*
6581	* Guest TR.
6582	*/
6583	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6584	{
6585	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6586
6587	/*
6588	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6589	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6590	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6591	*/
6592	uint16_t u16Sel;
6593	uint32_t u32Limit;
6594	uint64_t u64Base;
6595	uint32_t u32AccessRights;
6596	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6597	{
6598	u16Sel = pCtx->tr.Sel;
6599	u32Limit = pCtx->tr.u32Limit;
6600	u64Base = pCtx->tr.u64Base;
6601	u32AccessRights = pCtx->tr.Attr.u;
6602	}
6603	else
6604	{
6605	Assert(!pVmxTransient->fIsNestedGuest);
6606	Assert(pVM->hm.s.vmx.pRealModeTSS);
6607	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6608
6609	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6610	RTGCPHYS GCPhys;
6611	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6612	AssertRCReturn(rc, rc);
6613
6614	X86DESCATTR DescAttr;
6615	DescAttr.u = 0;
6616	DescAttr.n.u1Present = 1;
6617	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6618
6619	u16Sel = 0;
6620	u32Limit = HM_VTX_TSS_SIZE;
6621	u64Base = GCPhys;
6622	u32AccessRights = DescAttr.u;
6623	}
6624
6625	/* Validate. */
6626	Assert(!(u16Sel & RT_BIT(2)));
6627	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6628	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6629	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6630	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6631	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6632	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6633	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6634	Assert( (u32Limit & 0xfff) == 0xfff
6635	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6636	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6637	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6638
6639	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6640	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6641	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6642	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6643	AssertRCReturn(rc, rc);
6644
6645	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6646	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6647	}
6648
6649	/*
6650	* Guest GDTR.
6651	*/
6652	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6653	{
6654	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6655
6656	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6657	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6658	AssertRCReturn(rc, rc);
6659
6660	/* Validate. */
6661	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6662
6663	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6664	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6665	}
6666
6667	/*
6668	* Guest LDTR.
6669	*/
6670	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6671	{
6672	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6673
6674	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6675	uint32_t u32Access;
6676	if ( !pVmxTransient->fIsNestedGuest
6677	&& !pCtx->ldtr.Attr.u)
6678	u32Access = X86DESCATTR_UNUSABLE;
6679	else
6680	u32Access = pCtx->ldtr.Attr.u;
6681
6682	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6683	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6684	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6685	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6686	AssertRCReturn(rc, rc);
6687
6688	/* Validate. */
6689	if (!(u32Access & X86DESCATTR_UNUSABLE))
6690	{
6691	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6692	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6693	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6694	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6695	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6696	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6697	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6698	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6699	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6700	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6701	}
6702
6703	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6704	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6705	}
6706
6707	/*
6708	* Guest IDTR.
6709	*/
6710	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6711	{
6712	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6713
6714	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6715	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6716	AssertRCReturn(rc, rc);
6717
6718	/* Validate. */
6719	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6720
6721	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6722	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6723	}
6724
6725	return VINF_SUCCESS;
6726	}
6727
6728
6729	/**
6730	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6731	* areas.
6732	*
6733	* These MSRs will automatically be loaded to the host CPU on every successful
6734	* VM-entry and stored from the host CPU on every successful VM-exit.
6735	*
6736	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6737	* actual host MSR values are not- updated here for performance reasons. See
6738	* hmR0VmxExportHostMsrs().
6739	*
6740	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6741	*
6742	* @returns VBox status code.
6743	* @param pVCpu The cross context virtual CPU structure.
6744	* @param pVmxTransient The VMX-transient structure.
6745	*
6746	* @remarks No-long-jump zone!!!
6747	*/
6748	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6749	{
6750	AssertPtr(pVCpu);
6751	AssertPtr(pVmxTransient);
6752
6753	PVM pVM = pVCpu->CTX_SUFF(pVM);
6754	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6755
6756	/*
6757	* MSRs that we use the auto-load/store MSR area in the VMCS.
6758	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6759	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6760	*
6761	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6762	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6763	* emulation, nothing to do here.
6764	*/
6765	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6766	{
6767	if ( !pVmxTransient->fIsNestedGuest
6768	&& pVM->hm.s.fAllow64BitGuests)
6769	{
6770	#if HC_ARCH_BITS == 32
6771	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6772	Assert(!pVmxTransient->fIsNestedGuest);
6773
6774	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6775	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6776	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6777	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6778	AssertRCReturn(rc, rc);
6779	#endif
6780	}
6781	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6782	}
6783
6784	/*
6785	* Guest Sysenter MSRs.
6786	*/
6787	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6788	{
6789	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6790
6791	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6792	{
6793	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6794	AssertRCReturn(rc, rc);
6795	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6796	}
6797
6798	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6799	{
6800	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6801	AssertRCReturn(rc, rc);
6802	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6803	}
6804
6805	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6806	{
6807	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6808	AssertRCReturn(rc, rc);
6809	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6810	}
6811	}
6812
6813	/*
6814	* Guest/host EFER MSR.
6815	*/
6816	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6817	{
6818	/* Whether we are using the VMCS to swap the EFER MSR must have been
6819	determined earlier while exporting VM-entry/VM-exit controls. */
6820	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6821	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6822
6823	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6824	{
6825	/*
6826	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6827	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6828	*/
6829	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6830	{
6831	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6832	AssertRCReturn(rc, rc);
6833	}
6834	else
6835	{
6836	/*
6837	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6838	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6839	*/
6840	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6841	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6842	AssertRCReturn(rc, rc);
6843	}
6844	}
6845	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6846	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6847
6848	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6849	}
6850
6851	/*
6852	* Other MSRs.
6853	* Speculation Control (R/W).
6854	*/
6855	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6856	{
6857	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6858	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6859	{
6860	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6861	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6862	AssertRCReturn(rc, rc);
6863	}
6864	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6865	}
6866
6867	return VINF_SUCCESS;
6868	}
6869
6870
6871	/**
6872	* Selects up the appropriate function to run guest code.
6873	*
6874	* @returns VBox status code.
6875	* @param pVCpu The cross context virtual CPU structure.
6876	* @param pVmxTransient The VMX-transient structure.
6877	*
6878	* @remarks No-long-jump zone!!!
6879	*/
6880	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6881	{
6882	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6883	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6884
6885	if (CPUMIsGuestInLongModeEx(pCtx))
6886	{
6887	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6888	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6889	#endif
6890	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6891	#if HC_ARCH_BITS == 32
6892	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6893	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6894	{
6895	#ifdef VBOX_STRICT
6896	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6897	{
6898	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6899	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6900	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6901	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6902	("fCtxChanged=%#RX64\n", fCtxChanged));
6903	}
6904	#endif
6905	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6906
6907	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6908	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6909	pVmcsInfo->fSwitchedTo64on32 = true;
6910	Log4Func(("Selected 64-bit switcher\n"));
6911	}
6912	#else
6913	/* 64-bit host. */
6914	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6915	#endif
6916	}
6917	else
6918	{
6919	/* Guest is not in long mode, use the 32-bit handler. */
6920	#if HC_ARCH_BITS == 32
6921	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6922	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6923	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6924	{
6925	# ifdef VBOX_STRICT
6926	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6927	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6928	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6929	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6930	("fCtxChanged=%#RX64\n", fCtxChanged));
6931	# endif
6932	}
6933	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6934	/*
6935	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6936	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6937	* switcher flag now because we know the guest is in a sane state where it's safe
6938	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6939	* the much faster 32-bit switcher again.
6940	*/
6941	if (!pVmcsInfo->fSwitchedTo64on32)
6942	{
6943	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6944	Log4Func(("Selected 32-bit switcher\n"));
6945	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6946	}
6947	else
6948	{
6949	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6950	if ( pVmcsInfo->RealMode.fRealOnV86Active
6951	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6952	{
6953	pVmcsInfo->fSwitchedTo64on32 = false;
6954	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6956	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6957	\| HM_CHANGED_HOST_CONTEXT);
6958	Log4Func(("Selected 32-bit switcher (safe)\n"));
6959	}
6960	}
6961	# else
6962	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6963	# endif
6964	#else
6965	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6966	#endif
6967	}
6968	Assert(pVmcsInfo->pfnStartVM);
6969	return VINF_SUCCESS;
6970	}
6971
6972
6973	/**
6974	* Wrapper for running the guest code in VT-x.
6975	*
6976	* @returns VBox status code, no informational status codes.
6977	* @param pVCpu The cross context virtual CPU structure.
6978	* @param pVmxTransient The VMX-transient structure.
6979	*
6980	* @remarks No-long-jump zone!!!
6981	*/
6982	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6983	{
6984	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6985	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6986	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6987
6988	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6989
6990	/*
6991	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6992	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6993	* callee-saved and thus the need for this XMM wrapper.
6994	*
6995	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6996	*/
6997	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6998	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6999	PVM pVM = pVCpu->CTX_SUFF(pVM);
7000	#ifdef VBOX_WITH_KERNEL_USING_XMM
7001	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
7002	#else
7003	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
7004	#endif
7005	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
7006	return rc;
7007	}
7008
7009
7010	/**
7011	* Reports world-switch error and dumps some useful debug info.
7012	*
7013	* @param pVCpu The cross context virtual CPU structure.
7014	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
7015	* @param pVmxTransient The VMX-transient structure (only
7016	* exitReason updated).
7017	*/
7018	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
7019	{
7020	Assert(pVCpu);
7021	Assert(pVmxTransient);
7022	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7023
7024	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
7025	switch (rcVMRun)
7026	{
7027	case VERR_VMX_INVALID_VMXON_PTR:
7028	AssertFailed();
7029	break;
7030	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
7031	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
7032	{
7033	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
7034	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
7035	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
7036	AssertRC(rc);
7037
7038	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7039	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
7040	Cannot do it here as we may have been long preempted. */
7041
7042	#ifdef VBOX_STRICT
7043	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7044	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
7045	pVmxTransient->uExitReason));
7046	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
7047	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
7048	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
7049	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
7050	else
7051	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
7052	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
7053	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
7054
7055	/* VMX control bits. */
7056	uint32_t u32Val;
7057	uint64_t u64Val;
7058	RTHCUINTREG uHCReg;
7059	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
7060	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
7061	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
7062	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
7063	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
7064	{
7065	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
7066	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
7067	}
7068	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
7069	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
7070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
7071	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
7072	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
7073	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
7074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
7075	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
7076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
7077	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
7078	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
7079	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
7080	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
7081	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
7082	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
7083	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
7084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7085	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
7086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7087	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
7088	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
7089	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
7090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
7091	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
7092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
7093	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
7094	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7095	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7096	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7097	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7098	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7099	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7100	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7101	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7102	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7103	{
7104	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7105	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7106	}
7107
7108	/* Guest bits. */
7109	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
7110	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
7111	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
7112	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
7113	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
7114	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
7115	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
7116	{
7117	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
7118	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
7119	}
7120
7121	/* Host bits. */
7122	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
7123	Log4(("Host CR0 %#RHr\n", uHCReg));
7124	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
7125	Log4(("Host CR3 %#RHr\n", uHCReg));
7126	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
7127	Log4(("Host CR4 %#RHr\n", uHCReg));
7128
7129	RTGDTR HostGdtr;
7130	PCX86DESCHC pDesc;
7131	ASMGetGDTR(&HostGdtr);
7132	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
7133	Log4(("Host CS %#08x\n", u32Val));
7134	if (u32Val < HostGdtr.cbGdt)
7135	{
7136	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7137	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
7138	}
7139
7140	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
7141	Log4(("Host DS %#08x\n", u32Val));
7142	if (u32Val < HostGdtr.cbGdt)
7143	{
7144	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7145	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
7146	}
7147
7148	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
7149	Log4(("Host ES %#08x\n", u32Val));
7150	if (u32Val < HostGdtr.cbGdt)
7151	{
7152	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7153	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
7154	}
7155
7156	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
7157	Log4(("Host FS %#08x\n", u32Val));
7158	if (u32Val < HostGdtr.cbGdt)
7159	{
7160	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7161	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
7162	}
7163
7164	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
7165	Log4(("Host GS %#08x\n", u32Val));
7166	if (u32Val < HostGdtr.cbGdt)
7167	{
7168	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7169	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
7170	}
7171
7172	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
7173	Log4(("Host SS %#08x\n", u32Val));
7174	if (u32Val < HostGdtr.cbGdt)
7175	{
7176	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7177	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
7178	}
7179
7180	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
7181	Log4(("Host TR %#08x\n", u32Val));
7182	if (u32Val < HostGdtr.cbGdt)
7183	{
7184	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7185	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
7186	}
7187
7188	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
7189	Log4(("Host TR Base %#RHv\n", uHCReg));
7190	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
7191	Log4(("Host GDTR Base %#RHv\n", uHCReg));
7192	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
7193	Log4(("Host IDTR Base %#RHv\n", uHCReg));
7194	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
7195	Log4(("Host SYSENTER CS %#08x\n", u32Val));
7196	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
7197	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
7198	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
7199	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
7200	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
7201	Log4(("Host RSP %#RHv\n", uHCReg));
7202	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
7203	Log4(("Host RIP %#RHv\n", uHCReg));
7204	# if HC_ARCH_BITS == 64
7205	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
7206	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
7207	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
7208	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
7209	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
7210	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
7211	# endif
7212	#endif /* VBOX_STRICT */
7213	break;
7214	}
7215
7216	default:
7217	/* Impossible */
7218	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
7219	break;
7220	}
7221	}
7222
7223
7224	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
7225	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
7226	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
7227	# endif
7228
7229	/**
7230	* Initialize the VMCS-Read cache.
7231	*
7232	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
7233	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
7234	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
7235	* (those that have a 32-bit FULL & HIGH part).
7236	*
7237	* @param pVCpu The cross context virtual CPU structure.
7238	*/
7239	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
7240	{
7241	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
7242	do { \
7243	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
7244	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
7245	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
7246	++cReadFields; \
7247	} while (0)
7248
7249	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7250	uint32_t cReadFields = 0;
7251
7252	/*
7253	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
7254	* and serve to indicate exceptions to the rules.
7255	*/
7256
7257	/* Guest-natural selector base fields. */
7258	#if 0
7259	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
7260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
7261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
7262	#endif
7263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
7264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
7265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
7266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
7267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
7268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
7269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
7270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
7271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
7272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
7273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
7274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
7275	#if 0
7276	/* Unused natural width guest-state fields. */
7277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7279	#endif
7280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7282
7283	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7284	these 64-bit fields (using "FULL" and "HIGH" fields). */
7285	#if 0
7286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7295	#endif
7296
7297	/* Natural width guest-state fields. */
7298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7300
7301	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7302	{
7303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7304	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7305	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7306	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7307	}
7308	else
7309	{
7310	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7311	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7312	}
7313
7314	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7315	}
7316
7317
7318	/**
7319	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7320	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7321	* darwin, running 64-bit guests).
7322	*
7323	* @returns VBox status code.
7324	* @param pVCpu The cross context virtual CPU structure.
7325	* @param idxField The VMCS field encoding.
7326	* @param u64Val 16, 32 or 64-bit value.
7327	*/
7328	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7329	{
7330	AssertPtr(pVCpu);
7331	int rc;
7332	switch (idxField)
7333	{
7334	/*
7335	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7336	*/
7337	/* 64-bit Control fields. */
7338	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7339	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7340	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7341	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7342	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7343	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7344	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7345	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7346	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7347	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7348	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7349	case VMX_VMCS64_CTRL_EPTP_FULL:
7350	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7351	/* 64-bit Guest-state fields. */
7352	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7353	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7354	case VMX_VMCS64_GUEST_PAT_FULL:
7355	case VMX_VMCS64_GUEST_EFER_FULL:
7356	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7357	case VMX_VMCS64_GUEST_PDPTE0_FULL:
7358	case VMX_VMCS64_GUEST_PDPTE1_FULL:
7359	case VMX_VMCS64_GUEST_PDPTE2_FULL:
7360	case VMX_VMCS64_GUEST_PDPTE3_FULL:
7361	/* 64-bit Host-state fields. */
7362	case VMX_VMCS64_HOST_PAT_FULL:
7363	case VMX_VMCS64_HOST_EFER_FULL:
7364	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7365	{
7366	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7367	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7368	break;
7369	}
7370
7371	/*
7372	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7373	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7374	*/
7375	/* Natural-width Guest-state fields. */
7376	case VMX_VMCS_GUEST_CR3:
7377	case VMX_VMCS_GUEST_ES_BASE:
7378	case VMX_VMCS_GUEST_CS_BASE:
7379	case VMX_VMCS_GUEST_SS_BASE:
7380	case VMX_VMCS_GUEST_DS_BASE:
7381	case VMX_VMCS_GUEST_FS_BASE:
7382	case VMX_VMCS_GUEST_GS_BASE:
7383	case VMX_VMCS_GUEST_LDTR_BASE:
7384	case VMX_VMCS_GUEST_TR_BASE:
7385	case VMX_VMCS_GUEST_GDTR_BASE:
7386	case VMX_VMCS_GUEST_IDTR_BASE:
7387	case VMX_VMCS_GUEST_RSP:
7388	case VMX_VMCS_GUEST_RIP:
7389	case VMX_VMCS_GUEST_SYSENTER_ESP:
7390	case VMX_VMCS_GUEST_SYSENTER_EIP:
7391	{
7392	if (!(RT_HI_U32(u64Val)))
7393	{
7394	/* If this field is 64-bit, VT-x will zero out the top bits. */
7395	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7396	}
7397	else
7398	{
7399	/* Assert that only the 32->64 switcher case should ever come here. */
7400	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7401	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7402	}
7403	break;
7404	}
7405
7406	default:
7407	{
7408	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7409	pVCpu->hm.s.u32HMError = idxField;
7410	rc = VERR_INVALID_PARAMETER;
7411	break;
7412	}
7413	}
7414	AssertRCReturn(rc, rc);
7415	return rc;
7416	}
7417
7418
7419	/**
7420	* Queue up a VMWRITE by using the VMCS write cache.
7421	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7422	*
7423	* @param pVCpu The cross context virtual CPU structure.
7424	* @param idxField The VMCS field encoding.
7425	* @param u64Val 16, 32 or 64-bit value.
7426	*/
7427	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7428	{
7429	AssertPtr(pVCpu);
7430	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7431
7432	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7433	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7434
7435	/* Make sure there are no duplicates. */
7436	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7437	{
7438	if (pCache->Write.aField[i] == idxField)
7439	{
7440	pCache->Write.aFieldVal[i] = u64Val;
7441	return VINF_SUCCESS;
7442	}
7443	}
7444
7445	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7446	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7447	pCache->Write.cValidEntries++;
7448	return VINF_SUCCESS;
7449	}
7450	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7451
7452
7453	/**
7454	* Sets up the usage of TSC-offsetting and updates the VMCS.
7455	*
7456	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7457	* VMX-preemption timer.
7458	*
7459	* @returns VBox status code.
7460	* @param pVCpu The cross context virtual CPU structure.
7461	* @param pVmxTransient The VMX-transient structure.
7462	*
7463	* @remarks No-long-jump zone!!!
7464	*/
7465	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7466	{
7467	bool fOffsettedTsc;
7468	bool fParavirtTsc;
7469	uint64_t uTscOffset;
7470	PVM pVM = pVCpu->CTX_SUFF(pVM);
7471	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7472
7473	if (pVM->hm.s.vmx.fUsePreemptTimer)
7474	{
7475	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7476
7477	/* Make sure the returned values have sane upper and lower boundaries. */
7478	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7479	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7480	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7481	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7482
7483	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7484	* preemption timers here. We probably need to clamp the preemption timer,
7485	* after converting the timer value to the host. */
7486	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7487	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7488	AssertRC(rc);
7489	}
7490	else
7491	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7492
7493	if (fParavirtTsc)
7494	{
7495	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7496	information before every VM-entry, hence disable it for performance sake. */
7497	#if 0
7498	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7499	AssertRC(rc);
7500	#endif
7501	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7502	}
7503
7504	if ( fOffsettedTsc
7505	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7506	{
7507	if (pVmxTransient->fIsNestedGuest)
7508	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7509	hmR0VmxSetTscOffsetVmcs(pVCpu, pVmcsInfo, uTscOffset);
7510	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7511	}
7512	else
7513	{
7514	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7515	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7516	}
7517	}
7518
7519
7520	/**
7521	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7522	* VM-exit interruption info type.
7523	*
7524	* @returns The IEM exception flags.
7525	* @param uVector The event vector.
7526	* @param uVmxEventType The VMX event type.
7527	*
7528	* @remarks This function currently only constructs flags required for
7529	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7530	* and CR2 aspects of an exception are not included).
7531	*/
7532	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7533	{
7534	uint32_t fIemXcptFlags;
7535	switch (uVmxEventType)
7536	{
7537	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7538	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7539	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7540	break;
7541
7542	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7543	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7544	break;
7545
7546	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7547	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7548	break;
7549
7550	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7551	{
7552	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7553	if (uVector == X86_XCPT_BP)
7554	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7555	else if (uVector == X86_XCPT_OF)
7556	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7557	else
7558	{
7559	fIemXcptFlags = 0;
7560	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7561	}
7562	break;
7563	}
7564
7565	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7566	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7567	break;
7568
7569	default:
7570	fIemXcptFlags = 0;
7571	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7572	break;
7573	}
7574	return fIemXcptFlags;
7575	}
7576
7577
7578	/**
7579	* Sets an event as a pending event to be injected into the guest.
7580	*
7581	* @param pVCpu The cross context virtual CPU structure.
7582	* @param u32IntInfo The VM-entry interruption-information field.
7583	* @param cbInstr The VM-entry instruction length in bytes (for software
7584	* interrupts, exceptions and privileged software
7585	* exceptions).
7586	* @param u32ErrCode The VM-entry exception error code.
7587	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7588	* page-fault.
7589	*/
7590	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7591	RTGCUINTPTR GCPtrFaultAddress)
7592	{
7593	Assert(!pVCpu->hm.s.Event.fPending);
7594	pVCpu->hm.s.Event.fPending = true;
7595	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7596	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7597	pVCpu->hm.s.Event.cbInstr = cbInstr;
7598	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7599	}
7600
7601
7602	/**
7603	* Sets an external interrupt as pending-for-injection into the VM.
7604	*
7605	* @param pVCpu The cross context virtual CPU structure.
7606	* @param u8Interrupt The external interrupt vector.
7607	*/
7608	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7609	{
7610	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7611	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7612	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7613	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7614	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7615	}
7616
7617
7618	/**
7619	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7620	*
7621	* @param pVCpu The cross context virtual CPU structure.
7622	*/
7623	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7624	{
7625	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7626	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7627	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7628	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7629	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7630	}
7631
7632
7633	/**
7634	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7635	*
7636	* @param pVCpu The cross context virtual CPU structure.
7637	*/
7638	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7639	{
7640	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7641	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7642	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7643	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7644	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7645	}
7646
7647
7648	/**
7649	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7650	*
7651	* @param pVCpu The cross context virtual CPU structure.
7652	*/
7653	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7654	{
7655	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7657	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7658	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7659	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7660	}
7661
7662
7663	/**
7664	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7665	*
7666	* @param pVCpu The cross context virtual CPU structure.
7667	*/
7668	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7669	{
7670	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7672	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7673	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7674	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7675	}
7676
7677
7678	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7679	/**
7680	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7681	*
7682	* @param pVCpu The cross context virtual CPU structure.
7683	* @param u32ErrCode The error code for the general-protection exception.
7684	*/
7685	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7686	{
7687	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7689	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7690	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7691	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7692	}
7693
7694
7695	/**
7696	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7697	*
7698	* @param pVCpu The cross context virtual CPU structure.
7699	* @param u32ErrCode The error code for the stack exception.
7700	*/
7701	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7702	{
7703	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7704	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7706	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7707	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7708	}
7709	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7710
7711
7712	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7713	{
7714	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7715
7716	/*
7717	* If VT-x marks the segment as unusable, most other bits remain undefined:
7718	* - For CS the L, D and G bits have meaning.
7719	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7720	* - For the remaining data segments no bits are defined.
7721	*
7722	* The present bit and the unusable bit has been observed to be set at the
7723	* same time (the selector was supposed to be invalid as we started executing
7724	* a V8086 interrupt in ring-0).
7725	*
7726	* What should be important for the rest of the VBox code, is that the P bit is
7727	* cleared. Some of the other VBox code recognizes the unusable bit, but
7728	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7729	* safe side here, we'll strip off P and other bits we don't care about. If
7730	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7731	*
7732	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7733	*/
7734	#ifdef VBOX_STRICT
7735	uint32_t const uAttr = pSelReg->Attr.u;
7736	#endif
7737
7738	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7739	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7740	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7741
7742	#ifdef VBOX_STRICT
7743	VMMRZCallRing3Disable(pVCpu);
7744	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7745	# ifdef DEBUG_bird
7746	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7747	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7748	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7749	# endif
7750	VMMRZCallRing3Enable(pVCpu);
7751	NOREF(uAttr);
7752	#endif
7753	RT_NOREF2(pVCpu, idxSel);
7754	}
7755
7756
7757	/**
7758	* Imports a guest segment register from the current VMCS into the guest-CPU
7759	* context.
7760	*
7761	* @returns VBox status code.
7762	* @param pVCpu The cross context virtual CPU structure.
7763	* @param iSegReg The segment register number (X86_SREG_XXX).
7764	*
7765	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7766	* do not log!
7767	*/
7768	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7769	{
7770	Assert(iSegReg < X86_SREG_COUNT);
7771
7772	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7773	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7774	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7775	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7776	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7777	#else
7778	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7779	#endif
7780	uint64_t u64Base;
7781	uint32_t u32Sel, u32Limit, u32Attr;
7782	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7783	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7784	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7785	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7786	if (RT_SUCCESS(rc))
7787	{
7788	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7789	pSelReg->Sel = u32Sel;
7790	pSelReg->ValidSel = u32Sel;
7791	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7792	pSelReg->u32Limit = u32Limit;
7793	pSelReg->u64Base = u64Base;
7794	pSelReg->Attr.u = u32Attr;
7795	if (u32Attr & X86DESCATTR_UNUSABLE)
7796	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7797	}
7798	return rc;
7799	}
7800
7801
7802	/**
7803	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7804	*
7805	* @returns VBox status code.
7806	* @param pVCpu The cross context virtual CPU structure.
7807	*
7808	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7809	* do not log!
7810	*/
7811	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7812	{
7813	uint64_t u64Base;
7814	uint32_t u32Sel, u32Limit, u32Attr;
7815	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7816	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7817	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7818	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7819	if (RT_SUCCESS(rc))
7820	{
7821	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7822	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7823	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7824	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7825	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7826	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7827	if (u32Attr & X86DESCATTR_UNUSABLE)
7828	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7829	}
7830	return rc;
7831	}
7832
7833
7834	/**
7835	* Imports the guest TR from the current VMCS into the guest-CPU context.
7836	*
7837	* @returns VBox status code.
7838	* @param pVCpu The cross context virtual CPU structure.
7839	*
7840	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7841	* do not log!
7842	*/
7843	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7844	{
7845	uint32_t u32Sel, u32Limit, u32Attr;
7846	uint64_t u64Base;
7847	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7848	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7849	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7850	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7851	AssertRCReturn(rc, rc);
7852
7853	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7854	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7855	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7856	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7857	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7858	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7859	/* TR is the only selector that can never be unusable. */
7860	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7861	return VINF_SUCCESS;
7862	}
7863
7864
7865	/**
7866	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7867	*
7868	* @returns VBox status code.
7869	* @param pVCpu The cross context virtual CPU structure.
7870	*
7871	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7872	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7873	* instead!!!
7874	*/
7875	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7876	{
7877	uint64_t u64Val;
7878	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7879	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7880	{
7881	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7882	if (RT_SUCCESS(rc))
7883	{
7884	pCtx->rip = u64Val;
7885	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7886	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7887	}
7888	return rc;
7889	}
7890	return VINF_SUCCESS;
7891	}
7892
7893
7894	/**
7895	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7896	*
7897	* @returns VBox status code.
7898	* @param pVCpu The cross context virtual CPU structure.
7899	* @param pVmcsInfo The VMCS info. object.
7900	*
7901	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7902	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7903	* instead!!!
7904	*/
7905	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7906	{
7907	uint32_t u32Val;
7908	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7909	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7910	{
7911	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7912	if (RT_SUCCESS(rc))
7913	{
7914	pCtx->eflags.u32 = u32Val;
7915
7916	/* Restore eflags for real-on-v86-mode hack. */
7917	if (pVmcsInfo->RealMode.fRealOnV86Active)
7918	{
7919	pCtx->eflags.Bits.u1VM = 0;
7920	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7921	}
7922	}
7923	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7924	return rc;
7925	}
7926	return VINF_SUCCESS;
7927	}
7928
7929
7930	/**
7931	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7932	* context.
7933	*
7934	* @returns VBox status code.
7935	* @param pVCpu The cross context virtual CPU structure.
7936	* @param pVmcsInfo The VMCS info. object.
7937	*
7938	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7939	* do not log!
7940	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7941	* instead!!!
7942	*/
7943	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7944	{
7945	uint32_t u32Val;
7946	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7947	if (RT_SUCCESS(rc))
7948	{
7949	if (!u32Val)
7950	{
7951	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7952	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7953
7954	CPUMSetGuestNmiBlocking(pVCpu, false);
7955	}
7956	else
7957	{
7958	/*
7959	* We must import RIP here to set our EM interrupt-inhibited state.
7960	* We also import RFLAGS as our code that evaluates pending interrupts
7961	* before VM-entry requires it.
7962	*/
7963	rc = hmR0VmxImportGuestRip(pVCpu);
7964	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7965	if (RT_SUCCESS(rc))
7966	{
7967	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7968	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7969	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7970	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7971
7972	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7973	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7974	}
7975	}
7976	}
7977	return rc;
7978	}
7979
7980
7981	/**
7982	* Worker for VMXR0ImportStateOnDemand.
7983	*
7984	* @returns VBox status code.
7985	* @param pVCpu The cross context virtual CPU structure.
7986	* @param pVmcsInfo The VMCS info. object.
7987	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7988	*/
7989	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7990	{
7991	#define VMXLOCAL_BREAK_RC(a_rc) \
7992	if (RT_SUCCESS(a_rc)) \
7993	{ } \
7994	else \
7995	break
7996
7997	int rc = VINF_SUCCESS;
7998	PVM pVM = pVCpu->CTX_SUFF(pVM);
7999	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8000	uint64_t u64Val;
8001	uint32_t u32Val;
8002
8003	/*
8004	* Note! This is hack to workaround a mysterious BSOD observed with release builds
8005	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8006	* neither are other host platforms.
8007	*
8008	* Committing this temporarily as it prevents BSOD.
8009	*
8010	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8011	*/
8012	#ifdef RT_OS_WINDOWS
8013	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
8014	return VERR_HM_IPE_1;
8015	#endif
8016
8017	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8018
8019	/*
8020	* We disable interrupts to make the updating of the state and in particular
8021	* the fExtrn modification atomic wrt to preemption hooks.
8022	*/
8023	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8024
8025	fWhat &= pCtx->fExtrn;
8026	if (fWhat)
8027	{
8028	do
8029	{
8030	if (fWhat & CPUMCTX_EXTRN_RIP)
8031	{
8032	rc = hmR0VmxImportGuestRip(pVCpu);
8033	VMXLOCAL_BREAK_RC(rc);
8034	}
8035
8036	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8037	{
8038	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8039	VMXLOCAL_BREAK_RC(rc);
8040	}
8041
8042	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8043	{
8044	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8045	VMXLOCAL_BREAK_RC(rc);
8046	}
8047
8048	if (fWhat & CPUMCTX_EXTRN_RSP)
8049	{
8050	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8051	VMXLOCAL_BREAK_RC(rc);
8052	pCtx->rsp = u64Val;
8053	}
8054
8055	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8056	{
8057	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8058	if (fWhat & CPUMCTX_EXTRN_CS)
8059	{
8060	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8061	rc \|= hmR0VmxImportGuestRip(pVCpu);
8062	if (fRealOnV86Active)
8063	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8064	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8065	}
8066	if (fWhat & CPUMCTX_EXTRN_SS)
8067	{
8068	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8069	if (fRealOnV86Active)
8070	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8071	}
8072	if (fWhat & CPUMCTX_EXTRN_DS)
8073	{
8074	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8075	if (fRealOnV86Active)
8076	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8077	}
8078	if (fWhat & CPUMCTX_EXTRN_ES)
8079	{
8080	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8081	if (fRealOnV86Active)
8082	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8083	}
8084	if (fWhat & CPUMCTX_EXTRN_FS)
8085	{
8086	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8087	if (fRealOnV86Active)
8088	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8089	}
8090	if (fWhat & CPUMCTX_EXTRN_GS)
8091	{
8092	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8093	if (fRealOnV86Active)
8094	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8095	}
8096	VMXLOCAL_BREAK_RC(rc);
8097	}
8098
8099	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8100	{
8101	if (fWhat & CPUMCTX_EXTRN_LDTR)
8102	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
8103
8104	if (fWhat & CPUMCTX_EXTRN_GDTR)
8105	{
8106	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8107	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8108	pCtx->gdtr.pGdt = u64Val;
8109	pCtx->gdtr.cbGdt = u32Val;
8110	}
8111
8112	/* Guest IDTR. */
8113	if (fWhat & CPUMCTX_EXTRN_IDTR)
8114	{
8115	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8116	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8117	pCtx->idtr.pIdt = u64Val;
8118	pCtx->idtr.cbIdt = u32Val;
8119	}
8120
8121	/* Guest TR. */
8122	if (fWhat & CPUMCTX_EXTRN_TR)
8123	{
8124	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8125	don't need to import that one. */
8126	if (!pVmcsInfo->RealMode.fRealOnV86Active)
8127	rc \|= hmR0VmxImportGuestTr(pVCpu);
8128	}
8129	VMXLOCAL_BREAK_RC(rc);
8130	}
8131
8132	if (fWhat & CPUMCTX_EXTRN_DR7)
8133	{
8134	if (!pVCpu->hm.s.fUsingHyperDR7)
8135	{
8136	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8137	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8138	VMXLOCAL_BREAK_RC(rc);
8139	pCtx->dr[7] = u32Val;
8140	}
8141	}
8142
8143	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8144	{
8145	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8146	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8147	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8148	pCtx->SysEnter.cs = u32Val;
8149	VMXLOCAL_BREAK_RC(rc);
8150	}
8151
8152	#if HC_ARCH_BITS == 64
8153	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8154	{
8155	if ( pVM->hm.s.fAllow64BitGuests
8156	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8157	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8158	}
8159
8160	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8161	{
8162	if ( pVM->hm.s.fAllow64BitGuests
8163	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8164	{
8165	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8166	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8167	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8168	}
8169	}
8170	#endif
8171
8172	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
8173	#if HC_ARCH_BITS == 32
8174	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
8175	#endif
8176	)
8177	{
8178	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8179	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8180	Assert(pMsrs);
8181	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8182	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
8183	for (uint32_t i = 0; i < cMsrs; i++)
8184	{
8185	uint32_t const idMsr = pMsrs[i].u32Msr;
8186	switch (idMsr)
8187	{
8188	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8189	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8190	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8191	#if HC_ARCH_BITS == 32
8192	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8193	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8194	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8195	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8196	#endif
8197	default:
8198	{
8199	pCtx->fExtrn = 0;
8200	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8201	ASMSetFlags(fEFlags);
8202	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8203	return VERR_HM_UNEXPECTED_LD_ST_MSR;
8204	}
8205	}
8206	}
8207	}
8208
8209	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8210	{
8211	uint64_t u64Shadow;
8212	if (fWhat & CPUMCTX_EXTRN_CR0)
8213	{
8214	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8215	* remove when we drop 32-bit host w/ 64-bit host support, see
8216	* @bugref{9180#c39}. */
8217	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8218	#if HC_ARCH_BITS == 32
8219	uint32_t u32Shadow;
8220	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8221	u64Shadow = u32Shadow;
8222	#else
8223	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8224	#endif
8225	VMXLOCAL_BREAK_RC(rc);
8226	u64Val = u32Val;
8227	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8228	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
8229	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8230	/*
8231	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
8232	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8233	*/
8234	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8235	{
8236	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8237	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8238	}
8239	#endif
8240	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8241	CPUMSetGuestCR0(pVCpu, u64Val);
8242	VMMRZCallRing3Enable(pVCpu);
8243	}
8244
8245	if (fWhat & CPUMCTX_EXTRN_CR4)
8246	{
8247	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8248	* remove when we drop 32-bit host w/ 64-bit host support, see
8249	* @bugref{9180#c39}. */
8250	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8251	#if HC_ARCH_BITS == 32
8252	uint32_t u32Shadow;
8253	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8254	u64Shadow = u32Shadow;
8255	#else
8256	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8257	#endif
8258	VMXLOCAL_BREAK_RC(rc);
8259	u64Val = u32Val;
8260	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8261	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
8262	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8263	/*
8264	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
8265	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8266	*/
8267	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8268	{
8269	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8270	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8271	}
8272	#endif
8273	pCtx->cr4 = u64Val;
8274	}
8275
8276	if (fWhat & CPUMCTX_EXTRN_CR3)
8277	{
8278	/* CR0.PG bit changes are always intercepted, so it's up to date. */
8279	if ( pVM->hm.s.vmx.fUnrestrictedGuest
8280	\|\| ( pVM->hm.s.fNestedPaging
8281	&& CPUMIsGuestPagingEnabledEx(pCtx)))
8282	{
8283	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8284	VMXLOCAL_BREAK_RC(rc);
8285	if (pCtx->cr3 != u64Val)
8286	{
8287	pCtx->cr3 = u64Val;
8288	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8289	}
8290
8291	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8292	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8293	if (CPUMIsGuestInPAEModeEx(pCtx))
8294	{
8295	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8296	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8297	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8298	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8299	VMXLOCAL_BREAK_RC(rc);
8300	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8301	}
8302	}
8303	}
8304	}
8305
8306	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8307	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
8308	{
8309	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
8310	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
8311	{
8312	Assert(CPUMIsGuestInVmxRootMode(pCtx));
8313	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
8314	VMXLOCAL_BREAK_RC(rc);
8315	}
8316
8317	# if 0
8318	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8319	* VM-exit handlers as parameters. We would handle it differently when using
8320	* the fast path. */
8321	/*
8322	* The hardware virtualization state currently consists of VMCS fields that may be
8323	* modified by execution of the nested-guest (that are not part of the general
8324	* guest state) and is visible to guest software. Hence, it is technically part of
8325	* the guest-CPU state when executing a nested-guest.
8326	*/
8327	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8328	{
8329	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8330	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8331	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8332	VMXLOCAL_BREAK_RC(rc);
8333
8334	/*
8335	* VM-entry can fail due to invalid-guest state, machine-check events and
8336	* MSR loading failures. Other than VM-exit reason and Exit qualification
8337	* all other VMCS fields are left unmodified on VM-entry failure.
8338	*
8339	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8340	*/
8341	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8342	if (!fEntryFailed)
8343	{
8344	/*
8345	* Some notes on VMCS fields that may need importing when the fast path
8346	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8347	*
8348	* Requires fixing up when using hardware-assisted VMX:
8349	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8350	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
8351	* - IDT-vectoring info: Think about this.
8352	* - IDT-vectoring error code: Think about this.
8353	*
8354	* Emulated:
8355	* - Guest-interruptiblity state: Derived from FFs and RIP.
8356	* - Guest pending debug exceptions: Derived from DR6.
8357	* - Guest activity state: Emulated from EM state.
8358	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8359	* - Entry-interrupt info: Emulated, cleared to 0.
8360	*/
8361	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8362	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8363	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8364	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8365	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8366	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8367	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8368	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8369	/** @todo NSTVMX: Save and adjust preemption timer value. */
8370	}
8371
8372	VMXLOCAL_BREAK_RC(rc);
8373	}
8374	# endif
8375	}
8376	#endif
8377	} while (0);
8378
8379	if (RT_SUCCESS(rc))
8380	{
8381	/* Update fExtrn. */
8382	pCtx->fExtrn &= ~fWhat;
8383
8384	/* If everything has been imported, clear the HM keeper bit. */
8385	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8386	{
8387	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8388	Assert(!pCtx->fExtrn);
8389	}
8390	}
8391	}
8392	else
8393	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8394
8395	ASMSetFlags(fEFlags);
8396
8397	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8398
8399	if (RT_SUCCESS(rc))
8400	{ /* likely */ }
8401	else
8402	return rc;
8403
8404	/*
8405	* Honor any pending CR3 updates.
8406	*
8407	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8408	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8409	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8410	*
8411	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8412	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8413	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8414	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8415	*
8416	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8417	*/
8418	if (VMMRZCallRing3IsEnabled(pVCpu))
8419	{
8420	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8421	{
8422	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8423	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8424	}
8425
8426	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8427	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8428
8429	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8430	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8431	}
8432
8433	return VINF_SUCCESS;
8434	#undef VMXLOCAL_BREAK_RC
8435	}
8436
8437
8438	/**
8439	* Saves the guest state from the VMCS into the guest-CPU context.
8440	*
8441	* @returns VBox status code.
8442	* @param pVCpu The cross context virtual CPU structure.
8443	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8444	*/
8445	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8446	{
8447	AssertPtr(pVCpu);
8448	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8449	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8450	}
8451
8452
8453	/**
8454	* Check per-VM and per-VCPU force flag actions that require us to go back to
8455	* ring-3 for one reason or another.
8456	*
8457	* @returns Strict VBox status code (i.e. informational status codes too)
8458	* @retval VINF_SUCCESS if we don't have any actions that require going back to
8459	* ring-3.
8460	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8461	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8462	* interrupts)
8463	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8464	* all EMTs to be in ring-3.
8465	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8466	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8467	* to the EM loop.
8468	*
8469	* @param pVCpu The cross context virtual CPU structure.
8470	* @param fStepping Whether we are single-stepping the guest using the
8471	* hypervisor debugger.
8472	*/
8473	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8474	{
8475	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8476
8477	/*
8478	* Update pending interrupts into the APIC's IRR.
8479	*/
8480	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8481	APICUpdatePendingInterrupts(pVCpu);
8482
8483	/*
8484	* Anything pending? Should be more likely than not if we're doing a good job.
8485	*/
8486	PVM pVM = pVCpu->CTX_SUFF(pVM);
8487	if ( !fStepping
8488	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8489	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8490	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8491	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8492	return VINF_SUCCESS;
8493
8494	/* Pending PGM C3 sync. */
8495	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8496	{
8497	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8498	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8499	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8500	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8501	if (rcStrict2 != VINF_SUCCESS)
8502	{
8503	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8504	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8505	return rcStrict2;
8506	}
8507	}
8508
8509	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8510	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8511	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8512	{
8513	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8514	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8515	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8516	return rc2;
8517	}
8518
8519	/* Pending VM request packets, such as hardware interrupts. */
8520	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8521	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8522	{
8523	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
8524	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8525	return VINF_EM_PENDING_REQUEST;
8526	}
8527
8528	/* Pending PGM pool flushes. */
8529	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8530	{
8531	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
8532	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8533	return VINF_PGM_POOL_FLUSH_PENDING;
8534	}
8535
8536	/* Pending DMA requests. */
8537	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8538	{
8539	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
8540	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8541	return VINF_EM_RAW_TO_R3;
8542	}
8543
8544	return VINF_SUCCESS;
8545	}
8546
8547
8548	/**
8549	* Converts any TRPM trap into a pending HM event. This is typically used when
8550	* entering from ring-3 (not longjmp returns).
8551	*
8552	* @param pVCpu The cross context virtual CPU structure.
8553	*/
8554	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8555	{
8556	Assert(TRPMHasTrap(pVCpu));
8557	Assert(!pVCpu->hm.s.Event.fPending);
8558
8559	uint8_t uVector;
8560	TRPMEVENT enmTrpmEvent;
8561	RTGCUINT uErrCode;
8562	RTGCUINTPTR GCPtrFaultAddress;
8563	uint8_t cbInstr;
8564
8565	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8566	AssertRC(rc);
8567
8568	uint32_t u32IntInfo;
8569	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8570	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
8571
8572	rc = TRPMResetTrap(pVCpu);
8573	AssertRC(rc);
8574	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8575	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8576
8577	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8578	}
8579
8580
8581	/**
8582	* Converts the pending HM event into a TRPM trap.
8583	*
8584	* @param pVCpu The cross context virtual CPU structure.
8585	*/
8586	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8587	{
8588	Assert(pVCpu->hm.s.Event.fPending);
8589
8590	/* If a trap was already pending, we did something wrong! */
8591	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8592
8593	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8594	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8595	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8596
8597	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8598
8599	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8600	AssertRC(rc);
8601
8602	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8603	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8604
8605	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8606	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8607	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
8608	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8609
8610	/* We're now done converting the pending event. */
8611	pVCpu->hm.s.Event.fPending = false;
8612	}
8613
8614
8615	/**
8616	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8617	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8618	*
8619	* @param pVCpu The cross context virtual CPU structure.
8620	* @param pVmcsInfo The VMCS info. object.
8621	*/
8622	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8623	{
8624	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8625	{
8626	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8627	{
8628	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8629	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8630	AssertRC(rc);
8631	}
8632	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8633	}
8634
8635
8636	/**
8637	* Clears the interrupt-window exiting control in the VMCS.
8638	*
8639	* @param pVmcsInfo The VMCS info. object.
8640	*/
8641	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8642	{
8643	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8644	{
8645	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8646	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8647	}
8648	return VINF_SUCCESS;
8649	}
8650
8651
8652	/**
8653	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8654	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8655	*
8656	* @param pVCpu The cross context virtual CPU structure.
8657	* @param pVmcsInfo The VMCS info. object.
8658	*/
8659	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8660	{
8661	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8662	{
8663	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8664	{
8665	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8666	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8667	AssertRC(rc);
8668	Log4Func(("Setup NMI-window exiting\n"));
8669	}
8670	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8671	}
8672
8673
8674	/**
8675	* Clears the NMI-window exiting control in the VMCS.
8676	*
8677	* @param pVmcsInfo The VMCS info. object.
8678	*/
8679	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8680	{
8681	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8682	{
8683	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8684	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8685	}
8686	return VINF_SUCCESS;
8687	}
8688
8689
8690	/**
8691	* Does the necessary state syncing before returning to ring-3 for any reason
8692	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8693	*
8694	* @returns VBox status code.
8695	* @param pVCpu The cross context virtual CPU structure.
8696	* @param fImportState Whether to import the guest state from the VMCS back
8697	* to the guest-CPU context.
8698	*
8699	* @remarks No-long-jmp zone!!!
8700	*/
8701	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8702	{
8703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8704	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8705
8706	RTCPUID const idCpu = RTMpCpuId();
8707	Log4Func(("HostCpuId=%u\n", idCpu));
8708
8709	/*
8710	* !!! IMPORTANT !!!
8711	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8712	*/
8713
8714	/* Save the guest state if necessary. */
8715	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8716	if (fImportState)
8717	{
8718	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8719	AssertRCReturn(rc, rc);
8720	}
8721
8722	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8723	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8724	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8725
8726	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8727	#ifdef VBOX_STRICT
8728	if (CPUMIsHyperDebugStateActive(pVCpu))
8729	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8730	#endif
8731	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8732	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8733	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8734
8735	#if HC_ARCH_BITS == 64
8736	/* Restore host-state bits that VT-x only restores partially. */
8737	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8738	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8739	{
8740	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8741	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8742	}
8743	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8744	#endif
8745
8746	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8747	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8748	{
8749	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8750	if (!fImportState)
8751	{
8752	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8753	AssertRCReturn(rc, rc);
8754	}
8755	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8756	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8757	}
8758	else
8759	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8760
8761	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8762	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8763
8764	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8765	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8766	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8767	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8768	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8769	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8770	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8771	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8772	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8773	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8774
8775	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8776
8777	/** @todo This partially defeats the purpose of having preemption hooks.
8778	* The problem is, deregistering the hooks should be moved to a place that
8779	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8780	* context.
8781	*/
8782	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8783	AssertRCReturn(rc, rc);
8784
8785	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8786	/*
8787	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8788	* clear a shadow VMCS before allowing that VMCS to become active on another
8789	* logical processor. We may or may not be importing guest state which clears
8790	* it, so cover for it here.
8791	*
8792	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8793	*/
8794	if ( pVmcsInfo->pvShadowVmcs
8795	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8796	{
8797	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8798	AssertRCReturn(rc, rc);
8799	}
8800
8801	/*
8802	* Flag that we need to re-import the host state if we switch to this VMCS before
8803	* executing guest or nested-guest code.
8804	*/
8805	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8806	#endif
8807
8808	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8809	NOREF(idCpu);
8810	return VINF_SUCCESS;
8811	}
8812
8813
8814	/**
8815	* Leaves the VT-x session.
8816	*
8817	* @returns VBox status code.
8818	* @param pVCpu The cross context virtual CPU structure.
8819	*
8820	* @remarks No-long-jmp zone!!!
8821	*/
8822	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8823	{
8824	HM_DISABLE_PREEMPT(pVCpu);
8825	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8826	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8827	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8828
8829	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8830	and done this from the VMXR0ThreadCtxCallback(). */
8831	if (!pVCpu->hm.s.fLeaveDone)
8832	{
8833	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8834	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8835	pVCpu->hm.s.fLeaveDone = true;
8836	}
8837	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8838
8839	/*
8840	* !!! IMPORTANT !!!
8841	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8842	*/
8843
8844	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8845	/** @todo Deregistering here means we need to VMCLEAR always
8846	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8847	* for calling VMMR0ThreadCtxHookDisable here! */
8848	VMMR0ThreadCtxHookDisable(pVCpu);
8849
8850	/* Leave HM context. This takes care of local init (term). */
8851	int rc = HMR0LeaveCpu(pVCpu);
8852
8853	HM_RESTORE_PREEMPT();
8854	return rc;
8855	}
8856
8857
8858	/**
8859	* Does the necessary state syncing before doing a longjmp to ring-3.
8860	*
8861	* @returns VBox status code.
8862	* @param pVCpu The cross context virtual CPU structure.
8863	*
8864	* @remarks No-long-jmp zone!!!
8865	*/
8866	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8867	{
8868	return hmR0VmxLeaveSession(pVCpu);
8869	}
8870
8871
8872	/**
8873	* Take necessary actions before going back to ring-3.
8874	*
8875	* An action requires us to go back to ring-3. This function does the necessary
8876	* steps before we can safely return to ring-3. This is not the same as longjmps
8877	* to ring-3, this is voluntary and prepares the guest so it may continue
8878	* executing outside HM (recompiler/IEM).
8879	*
8880	* @returns VBox status code.
8881	* @param pVCpu The cross context virtual CPU structure.
8882	* @param rcExit The reason for exiting to ring-3. Can be
8883	* VINF_VMM_UNKNOWN_RING3_CALL.
8884	*/
8885	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8886	{
8887	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8888
8889	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8890	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8891	{
8892	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8893	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8894	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8895	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8896	}
8897
8898	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8899	VMMRZCallRing3Disable(pVCpu);
8900	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8901
8902	/*
8903	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8904	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8905	*
8906	* This is because execution may continue from ring-3 and we would need to inject
8907	* the event from there (hence place it back in TRPM).
8908	*/
8909	if (pVCpu->hm.s.Event.fPending)
8910	{
8911	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8912	Assert(!pVCpu->hm.s.Event.fPending);
8913
8914	/* Clear the events from the VMCS. */
8915	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8916	AssertRCReturn(rc, rc);
8917	}
8918	#ifdef VBOX_STRICT
8919	else
8920	{
8921	/*
8922	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8923	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8924	* occasionally, see @bugref{9180#c42}.
8925	*
8926	* However, if the VM-entry failed, any VM entry-interruption info. field would
8927	* be left unmodified as the event would not have been injected to the guest. In
8928	* such cases, don't assert, we're not going to continue guest execution anyway.
8929	*/
8930	uint32_t uExitReason;
8931	uint32_t uEntryIntInfo;
8932	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8933	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8934	AssertRC(rc);
8935	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8936	}
8937	#endif
8938
8939	/*
8940	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8941	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8942	* (e.g. TPR below threshold).
8943	*/
8944	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8945	{
8946	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8947	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8948	AssertRCReturn(rc, rc);
8949	}
8950
8951	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8952	and if we're injecting an event we should have a TRPM trap pending. */
8953	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8954	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8955	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8956	#endif
8957
8958	/* Save guest state and restore host state bits. */
8959	int rc = hmR0VmxLeaveSession(pVCpu);
8960	AssertRCReturn(rc, rc);
8961	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8962
8963	/* Thread-context hooks are unregistered at this point!!! */
8964
8965	/* Sync recompiler state. */
8966	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8967	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8968	\| CPUM_CHANGED_LDTR
8969	\| CPUM_CHANGED_GDTR
8970	\| CPUM_CHANGED_IDTR
8971	\| CPUM_CHANGED_TR
8972	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8973	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8974	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8975	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8976
8977	Assert(!pVCpu->hm.s.fClearTrapFlag);
8978
8979	/* Update the exit-to-ring 3 reason. */
8980	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8981
8982	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8983	if ( rcExit != VINF_EM_RAW_INTERRUPT
8984	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8985	{
8986	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8987	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8988	}
8989
8990	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8991
8992	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8993	VMMRZCallRing3RemoveNotification(pVCpu);
8994	VMMRZCallRing3Enable(pVCpu);
8995
8996	return rc;
8997	}
8998
8999
9000	/**
9001	* VMMRZCallRing3() callback wrapper which saves the guest state before we
9002	* longjump to ring-3 and possibly get preempted.
9003	*
9004	* @returns VBox status code.
9005	* @param pVCpu The cross context virtual CPU structure.
9006	* @param enmOperation The operation causing the ring-3 longjump.
9007	* @param pvUser User argument, currently unused, NULL.
9008	*/
9009	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9010	{
9011	RT_NOREF(pvUser);
9012	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9013	{
9014	/*
9015	* !!! IMPORTANT !!!
9016	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9017	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9018	*/
9019	VMMRZCallRing3RemoveNotification(pVCpu);
9020	VMMRZCallRing3Disable(pVCpu);
9021	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9022	RTThreadPreemptDisable(&PreemptState);
9023
9024	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9025	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9026	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9027	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9028
9029	#if HC_ARCH_BITS == 64
9030	/* Restore host-state bits that VT-x only restores partially. */
9031	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9032	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9033	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9034	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9035	#endif
9036
9037	/* Restore the lazy host MSRs as we're leaving VT-x context. */
9038	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9039	hmR0VmxLazyRestoreHostMsrs(pVCpu);
9040
9041	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9042	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9043	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9044
9045	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
9046	cleared as part of importing the guest state above. */
9047	hmR0VmxClearVmcs(pVmcsInfo);
9048
9049	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9050	VMMR0ThreadCtxHookDisable(pVCpu);
9051	HMR0LeaveCpu(pVCpu);
9052	RTThreadPreemptRestore(&PreemptState);
9053	return VINF_SUCCESS;
9054	}
9055
9056	Assert(pVCpu);
9057	Assert(pvUser);
9058	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9059	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9060
9061	VMMRZCallRing3Disable(pVCpu);
9062	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9063
9064	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9065
9066	int rc = hmR0VmxLongJmpToRing3(pVCpu);
9067	AssertRCReturn(rc, rc);
9068
9069	VMMRZCallRing3Enable(pVCpu);
9070	return VINF_SUCCESS;
9071	}
9072
9073
9074	/**
9075	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9076	* stack.
9077	*
9078	* @returns Strict VBox status code (i.e. informational status codes too).
9079	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9080	* @param pVCpu The cross context virtual CPU structure.
9081	* @param uValue The value to push to the guest stack.
9082	*/
9083	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9084	{
9085	/*
9086	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9087	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9088	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9089	*/
9090	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9091	if (pCtx->sp == 1)
9092	return VINF_EM_RESET;
9093	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9094	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9095	AssertRC(rc);
9096	return rc;
9097	}
9098
9099
9100	/**
9101	* Injects an event into the guest upon VM-entry by updating the relevant fields
9102	* in the VM-entry area in the VMCS.
9103	*
9104	* @returns Strict VBox status code (i.e. informational status codes too).
9105	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9106	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9107	*
9108	* @param pVCpu The cross context virtual CPU structure.
9109	* @param pVmxTransient The VMX-transient structure.
9110	* @param pEvent The event being injected.
9111	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
9112	* will be updated if necessary. This cannot not be NULL.
9113	* @param fStepping Whether we're single-stepping guest execution and should
9114	* return VINF_EM_DBG_STEPPED if the event is injected
9115	* directly (registers modified by us, not by hardware on
9116	* VM-entry).
9117	*/
9118	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9119	uint32_t *pfIntrState)
9120	{
9121	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9122	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9123	Assert(pfIntrState);
9124
9125	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9126	uint32_t u32IntInfo = pEvent->u64IntInfo;
9127	uint32_t const u32ErrCode = pEvent->u32ErrCode;
9128	uint32_t const cbInstr = pEvent->cbInstr;
9129	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9130	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9131	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9132
9133	#ifdef VBOX_STRICT
9134	/*
9135	* Validate the error-code-valid bit for hardware exceptions.
9136	* No error codes for exceptions in real-mode.
9137	*
9138	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9139	*/
9140	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9141	&& !CPUMIsGuestInRealModeEx(pCtx))
9142	{
9143	switch (uVector)
9144	{
9145	case X86_XCPT_PF:
9146	case X86_XCPT_DF:
9147	case X86_XCPT_TS:
9148	case X86_XCPT_NP:
9149	case X86_XCPT_SS:
9150	case X86_XCPT_GP:
9151	case X86_XCPT_AC:
9152	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9153	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9154	RT_FALL_THRU();
9155	default:
9156	break;
9157	}
9158	}
9159
9160	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
9161	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9162	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9163	#endif
9164
9165	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9166
9167	/*
9168	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
9169	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9170	* interrupt handler in the (real-mode) guest.
9171	*
9172	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9173	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9174	*/
9175	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9176	{
9177	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9178	{
9179	/*
9180	* For CPUs with unrestricted guest execution enabled and with the guest
9181	* in real-mode, we must not set the deliver-error-code bit.
9182	*
9183	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9184	*/
9185	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9186	}
9187	else
9188	{
9189	PVM pVM = pVCpu->CTX_SUFF(pVM);
9190	Assert(PDMVmmDevHeapIsEnabled(pVM));
9191	Assert(pVM->hm.s.vmx.pRealModeTSS);
9192	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9193
9194	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9195	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9196	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
9197	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
9198	AssertRCReturn(rc2, rc2);
9199
9200	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9201	size_t const cbIdtEntry = sizeof(X86IDTR16);
9202	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9203	{
9204	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9205	if (uVector == X86_XCPT_DF)
9206	return VINF_EM_RESET;
9207
9208	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9209	No error codes for exceptions in real-mode. */
9210	if (uVector == X86_XCPT_GP)
9211	{
9212	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9213	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9214	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9215	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9216	HMEVENT EventXcptDf;
9217	RT_ZERO(EventXcptDf);
9218	EventXcptDf.u64IntInfo = uXcptDfInfo;
9219	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9220	}
9221
9222	/*
9223	* If we're injecting an event with no valid IDT entry, inject a #GP.
9224	* No error codes for exceptions in real-mode.
9225	*
9226	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9227	*/
9228	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9229	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9230	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9231	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9232	HMEVENT EventXcptGp;
9233	RT_ZERO(EventXcptGp);
9234	EventXcptGp.u64IntInfo = uXcptGpInfo;
9235	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9236	}
9237
9238	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9239	uint16_t uGuestIp = pCtx->ip;
9240	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9241	{
9242	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9243	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
9244	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9245	}
9246	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9247	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9248
9249	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9250	X86IDTR16 IdtEntry;
9251	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9252	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9253	AssertRCReturn(rc2, rc2);
9254
9255	/* Construct the stack frame for the interrupt/exception handler. */
9256	VBOXSTRICTRC rcStrict;
9257	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9258	if (rcStrict == VINF_SUCCESS)
9259	{
9260	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9261	if (rcStrict == VINF_SUCCESS)
9262	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9263	}
9264
9265	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
9266	if (rcStrict == VINF_SUCCESS)
9267	{
9268	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
9269	pCtx->rip = IdtEntry.offSel;
9270	pCtx->cs.Sel = IdtEntry.uSel;
9271	pCtx->cs.ValidSel = IdtEntry.uSel;
9272	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9273	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9274	&& uVector == X86_XCPT_PF)
9275	pCtx->cr2 = GCPtrFault;
9276
9277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
9278	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9279	\| HM_CHANGED_GUEST_RSP);
9280
9281	/*
9282	* If we delivered a hardware exception (other than an NMI) and if there was
9283	* block-by-STI in effect, we should clear it.
9284	*/
9285	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9286	{
9287	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9288	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9289	Log4Func(("Clearing inhibition due to STI\n"));
9290	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9291	}
9292
9293	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9294	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9295
9296	/*
9297	* The event has been truly dispatched to the guest. Mark it as no longer pending so
9298	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9299	*/
9300	pVCpu->hm.s.Event.fPending = false;
9301
9302	/*
9303	* If we eventually support nested-guest execution without unrestricted guest execution,
9304	* we should set fInterceptEvents here.
9305	*/
9306	Assert(!pVmxTransient->fIsNestedGuest);
9307
9308	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9309	if (fStepping)
9310	rcStrict = VINF_EM_DBG_STEPPED;
9311	}
9312	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9313	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9314	return rcStrict;
9315	}
9316	}
9317
9318	/*
9319	* Validate.
9320	*/
9321	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9322	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9323
9324	/*
9325	* Inject the event into the VMCS.
9326	*/
9327	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9328	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9329	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9330	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9331	AssertRCReturn(rc, rc);
9332
9333	/*
9334	* Update guest CR2 if this is a page-fault.
9335	*/
9336	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
9337	pCtx->cr2 = GCPtrFault;
9338
9339	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9340	return VINF_SUCCESS;
9341	}
9342
9343
9344	/**
9345	* Evaluates the event to be delivered to the guest and sets it as the pending
9346	* event.
9347	*
9348	* @returns Strict VBox status code (i.e. informational status codes too).
9349	* @param pVCpu The cross context virtual CPU structure.
9350	* @param pVmxTransient The VMX-transient structure.
9351	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
9352	*/
9353	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9354	{
9355	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9356	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9357	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
9358
9359	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9360	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9361	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9362	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9363	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9364
9365	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9366	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9367	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9368	Assert(!TRPMHasTrap(pVCpu));
9369	Assert(pfIntrState);
9370
9371	*pfIntrState = fIntrState;
9372
9373	/*
9374	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9375	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9376	*/
9377	/** @todo SMI. SMIs take priority over NMIs. */
9378	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9379	{
9380	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9381	if ( !pVCpu->hm.s.Event.fPending
9382	&& !fBlockNmi
9383	&& !fBlockSti
9384	&& !fBlockMovSS)
9385	{
9386	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9387	if ( fIsNestedGuest
9388	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9389	return IEMExecVmxVmexitXcptNmi(pVCpu);
9390	#endif
9391	hmR0VmxSetPendingXcptNmi(pVCpu);
9392	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9393	Log4Func(("Pending NMI\n"));
9394	}
9395	else if (!fIsNestedGuest)
9396	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9397	}
9398	/*
9399	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9400	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9401	*/
9402	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
9403	&& !pVCpu->hm.s.fSingleInstruction)
9404	{
9405	Assert(!DBGFIsStepping(pVCpu));
9406	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9407	AssertRCReturn(rc, rc);
9408	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9409	if ( !pVCpu->hm.s.Event.fPending
9410	&& !fBlockInt
9411	&& !fBlockSti
9412	&& !fBlockMovSS)
9413	{
9414	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9415	if ( fIsNestedGuest
9416	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9417	{
9418	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
9419	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9420	return rcStrict;
9421	}
9422	#endif
9423	uint8_t u8Interrupt;
9424	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9425	if (RT_SUCCESS(rc))
9426	{
9427	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9428	if ( fIsNestedGuest
9429	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9430	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9431	{
9432	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9433	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9434	return rcStrict;
9435	}
9436	#endif
9437	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9438	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9439	}
9440	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9441	{
9442	if ( !fIsNestedGuest
9443	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9444	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9445	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9446
9447	/*
9448	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9449	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9450	* need to re-set this force-flag here.
9451	*/
9452	}
9453	else
9454	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9455	}
9456	else if (!fIsNestedGuest)
9457	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9458	}
9459
9460	return VINF_SUCCESS;
9461	}
9462
9463
9464	/**
9465	* Injects any pending events into the guest if the guest is in a state to
9466	* receive them.
9467	*
9468	* @returns Strict VBox status code (i.e. informational status codes too).
9469	* @param pVCpu The cross context virtual CPU structure.
9470	* @param pVmxTransient The VMX-transient structure.
9471	* @param fIntrState The VT-x guest-interruptibility state.
9472	* @param fStepping Whether we are single-stepping the guest using the
9473	* hypervisor debugger and should return
9474	* VINF_EM_DBG_STEPPED if the event was dispatched
9475	* directly.
9476	*/
9477	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9478	{
9479	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9480	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9481
9482	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9483	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9484
9485	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9486	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9487	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9488	Assert(!TRPMHasTrap(pVCpu));
9489
9490	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9491	if (pVCpu->hm.s.Event.fPending)
9492	{
9493	/*
9494	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9495	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9496	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9497	*
9498	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9499	*/
9500	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9501	#ifdef VBOX_STRICT
9502	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9503	{
9504	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9505	Assert(!fBlockInt);
9506	Assert(!fBlockSti);
9507	Assert(!fBlockMovSS);
9508	}
9509	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9510	{
9511	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9512	Assert(!fBlockSti);
9513	Assert(!fBlockMovSS);
9514	Assert(!fBlockNmi);
9515	}
9516	#endif
9517	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9518	uIntType));
9519
9520	/*
9521	* Inject the event and get any changes to the guest-interruptibility state.
9522	*
9523	* The guest-interruptibility state may need to be updated if we inject the event
9524	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9525	*/
9526	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9527	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9528
9529	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9530	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9531	else
9532	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9533	}
9534
9535	/*
9536	* Update the guest-interruptibility state.
9537	*
9538	* This is required for the real-on-v86 software interrupt injection case above, as well as
9539	* updates to the guest state from ring-3 or IEM/REM.
9540	*/
9541	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9542	AssertRCReturn(rc, rc);
9543
9544	/*
9545	* There's no need to clear the VM-entry interruption-information field here if we're not
9546	* injecting anything. VT-x clears the valid bit on every VM-exit.
9547	*
9548	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9549	*/
9550
9551	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9552	NOREF(fBlockMovSS); NOREF(fBlockSti);
9553	return rcStrict;
9554	}
9555
9556
9557	/**
9558	* Enters the VT-x session.
9559	*
9560	* @returns VBox status code.
9561	* @param pVCpu The cross context virtual CPU structure.
9562	*/
9563	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9564	{
9565	AssertPtr(pVCpu);
9566	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9567	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9568
9569	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9570	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9571	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9572
9573	#ifdef VBOX_STRICT
9574	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9575	RTCCUINTREG uHostCr4 = ASMGetCR4();
9576	if (!(uHostCr4 & X86_CR4_VMXE))
9577	{
9578	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9579	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9580	}
9581	#endif
9582
9583	/*
9584	* Load the appropriate VMCS as the current and active one.
9585	*/
9586	PVMXVMCSINFO pVmcsInfo;
9587	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9588	if (!fInNestedGuestMode)
9589	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9590	else
9591	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9592	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9593	if (RT_SUCCESS(rc))
9594	{
9595	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9596	pVCpu->hm.s.fLeaveDone = false;
9597	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9598
9599	/*
9600	* Do the EMT scheduled L1D flush here if needed.
9601	*/
9602	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9603	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9604	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9605	hmR0MdsClear();
9606	}
9607	return rc;
9608	}
9609
9610
9611	/**
9612	* The thread-context callback (only on platforms which support it).
9613	*
9614	* @param enmEvent The thread-context event.
9615	* @param pVCpu The cross context virtual CPU structure.
9616	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9617	* @thread EMT(pVCpu)
9618	*/
9619	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9620	{
9621	AssertPtr(pVCpu);
9622	RT_NOREF1(fGlobalInit);
9623
9624	switch (enmEvent)
9625	{
9626	case RTTHREADCTXEVENT_OUT:
9627	{
9628	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9629	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9630	VMCPU_ASSERT_EMT(pVCpu);
9631
9632	/* No longjmps (logger flushes, locks) in this fragile context. */
9633	VMMRZCallRing3Disable(pVCpu);
9634	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9635
9636	/* Restore host-state (FPU, debug etc.) */
9637	if (!pVCpu->hm.s.fLeaveDone)
9638	{
9639	/*
9640	* Do -not- import the guest-state here as we might already be in the middle of importing
9641	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9642	*/
9643	hmR0VmxLeave(pVCpu, false /* fImportState */);
9644	pVCpu->hm.s.fLeaveDone = true;
9645	}
9646
9647	/* Leave HM context, takes care of local init (term). */
9648	int rc = HMR0LeaveCpu(pVCpu);
9649	AssertRC(rc);
9650
9651	/* Restore longjmp state. */
9652	VMMRZCallRing3Enable(pVCpu);
9653	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9654	break;
9655	}
9656
9657	case RTTHREADCTXEVENT_IN:
9658	{
9659	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9660	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9661	VMCPU_ASSERT_EMT(pVCpu);
9662
9663	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9664	VMMRZCallRing3Disable(pVCpu);
9665	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9666
9667	/* Initialize the bare minimum state required for HM. This takes care of
9668	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9669	int rc = hmR0EnterCpu(pVCpu);
9670	AssertRC(rc);
9671	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9672	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9673
9674	/* Load the active VMCS as the current one. */
9675	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9676	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9677	AssertRC(rc);
9678	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9679	pVCpu->hm.s.fLeaveDone = false;
9680
9681	/* Do the EMT scheduled L1D flush if needed. */
9682	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9683	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9684
9685	/* Restore longjmp state. */
9686	VMMRZCallRing3Enable(pVCpu);
9687	break;
9688	}
9689
9690	default:
9691	break;
9692	}
9693	}
9694
9695
9696	/**
9697	* Exports the host state into the VMCS host-state area.
9698	* Sets up the VM-exit MSR-load area.
9699	*
9700	* The CPU state will be loaded from these fields on every successful VM-exit.
9701	*
9702	* @returns VBox status code.
9703	* @param pVCpu The cross context virtual CPU structure.
9704	*
9705	* @remarks No-long-jump zone!!!
9706	*/
9707	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9708	{
9709	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9710
9711	int rc = VINF_SUCCESS;
9712	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9713	{
9714	rc = hmR0VmxExportHostControlRegs();
9715	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9716
9717	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9718	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9719
9720	rc = hmR0VmxExportHostMsrs(pVCpu);
9721	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9722
9723	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9724	}
9725	return rc;
9726	}
9727
9728
9729	/**
9730	* Saves the host state in the VMCS host-state.
9731	*
9732	* @returns VBox status code.
9733	* @param pVCpu The cross context virtual CPU structure.
9734	*
9735	* @remarks No-long-jump zone!!!
9736	*/
9737	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9738	{
9739	AssertPtr(pVCpu);
9740	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9741
9742	/*
9743	* Export the host state here while entering HM context.
9744	* When thread-context hooks are used, we might get preempted and have to re-save the host
9745	* state but most of the time we won't be, so do it here before we disable interrupts.
9746	*/
9747	return hmR0VmxExportHostState(pVCpu);
9748	}
9749
9750
9751	/**
9752	* Exports the guest state into the VMCS guest-state area.
9753	*
9754	* The will typically be done before VM-entry when the guest-CPU state and the
9755	* VMCS state may potentially be out of sync.
9756	*
9757	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9758	* VM-entry controls.
9759	* Sets up the appropriate VMX non-root function to execute guest code based on
9760	* the guest CPU mode.
9761	*
9762	* @returns VBox strict status code.
9763	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9764	* without unrestricted guest execution and the VMMDev is not presently
9765	* mapped (e.g. EFI32).
9766	*
9767	* @param pVCpu The cross context virtual CPU structure.
9768	* @param pVmxTransient The VMX-transient structure.
9769	*
9770	* @remarks No-long-jump zone!!!
9771	*/
9772	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9773	{
9774	AssertPtr(pVCpu);
9775	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9776	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9777
9778	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9779
9780	/*
9781	* Determine real-on-v86 mode.
9782	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9783	*/
9784	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9785	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9786	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9787	pVmcsInfo->RealMode. fRealOnV86Active = false;
9788	else
9789	{
9790	Assert(!pVmxTransient->fIsNestedGuest);
9791	pVmcsInfo->RealMode.fRealOnV86Active = true;
9792	}
9793
9794	/*
9795	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9796	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9797	*/
9798	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9799	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9800	* be a need to evaluate this everytime since I'm pretty sure we intercept
9801	* all guest paging mode changes. */
9802	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9803	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9804
9805	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9806	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9807
9808	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9809	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9810
9811	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9812	if (rcStrict == VINF_SUCCESS)
9813	{ /* likely */ }
9814	else
9815	{
9816	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9817	return rcStrict;
9818	}
9819
9820	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9821	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9822
9823	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9824	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9825
9826	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9827	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9828
9829	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9830	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9831
9832	rc = hmR0VmxExportGuestRip(pVCpu);
9833	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9834	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9835	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9836
9837	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9838	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9839
9840	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9841	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9842	\| HM_CHANGED_GUEST_CR2
9843	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9844	\| HM_CHANGED_GUEST_X87
9845	\| HM_CHANGED_GUEST_SSE_AVX
9846	\| HM_CHANGED_GUEST_OTHER_XSAVE
9847	\| HM_CHANGED_GUEST_XCRx
9848	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9849	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9850	\| HM_CHANGED_GUEST_TSC_AUX
9851	\| HM_CHANGED_GUEST_OTHER_MSRS
9852	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9853
9854	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9855	return rc;
9856	}
9857
9858
9859	/**
9860	* Exports the state shared between the host and guest into the VMCS.
9861	*
9862	* @param pVCpu The cross context virtual CPU structure.
9863	* @param pVmxTransient The VMX-transient structure.
9864	*
9865	* @remarks No-long-jump zone!!!
9866	*/
9867	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9868	{
9869	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9870	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9871
9872	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9873	{
9874	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9875	AssertRC(rc);
9876	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9877
9878	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9879	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9880	{
9881	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9882	AssertRC(rc);
9883	}
9884	}
9885
9886	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9887	{
9888	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9889	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9890	}
9891
9892	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9893	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9894	}
9895
9896
9897	/**
9898	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9899	*
9900	* @returns Strict VBox status code (i.e. informational status codes too).
9901	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9902	* without unrestricted guest execution and the VMMDev is not presently
9903	* mapped (e.g. EFI32).
9904	*
9905	* @param pVCpu The cross context virtual CPU structure.
9906	* @param pVmxTransient The VMX-transient structure.
9907	*
9908	* @remarks No-long-jump zone!!!
9909	*/
9910	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9911	{
9912	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9913	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9914	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9915
9916	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9917	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9918	#endif
9919
9920	/*
9921	* For many exits it's only RIP that changes and hence try to export it first
9922	* without going through a lot of change flag checks.
9923	*/
9924	VBOXSTRICTRC rcStrict;
9925	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9926	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9927	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9928	{
9929	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9930	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9931	{ /* likely */}
9932	else
9933	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9935	}
9936	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9937	{
9938	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9939	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9940	{ /* likely */}
9941	else
9942	{
9943	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9944	VBOXSTRICTRC_VAL(rcStrict)));
9945	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9946	return rcStrict;
9947	}
9948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9949	}
9950	else
9951	rcStrict = VINF_SUCCESS;
9952
9953	#ifdef VBOX_STRICT
9954	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9955	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9956	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9957	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9958	("fCtxChanged=%#RX64\n", fCtxChanged));
9959	#endif
9960	return rcStrict;
9961	}
9962
9963
9964	/**
9965	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9966	* and update error record fields accordingly.
9967	*
9968	* @returns VMX_IGS_* error codes.
9969	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9970	* wrong with the guest state.
9971	*
9972	* @param pVCpu The cross context virtual CPU structure.
9973	* @param pVmcsInfo The VMCS info. object.
9974	*
9975	* @remarks This function assumes our cache of the VMCS controls
9976	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9977	*/
9978	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9979	{
9980	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9981	#define HMVMX_CHECK_BREAK(expr, err) do { \
9982	if (!(expr)) { uError = (err); break; } \
9983	} while (0)
9984
9985	int rc;
9986	PVM pVM = pVCpu->CTX_SUFF(pVM);
9987	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9988	uint32_t uError = VMX_IGS_ERROR;
9989	uint32_t u32Val;
9990	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9991
9992	do
9993	{
9994	/*
9995	* CR0.
9996	*/
9997	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9998	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9999	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10000	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10001	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10002	if (fUnrestrictedGuest)
10003	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10004
10005	uint32_t u32GuestCr0;
10006	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10007	AssertRCBreak(rc);
10008	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10009	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10010	if ( !fUnrestrictedGuest
10011	&& (u32GuestCr0 & X86_CR0_PG)
10012	&& !(u32GuestCr0 & X86_CR0_PE))
10013	{
10014	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10015	}
10016
10017	/*
10018	* CR4.
10019	*/
10020	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10021	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10022	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10023
10024	uint32_t u32GuestCr4;
10025	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10026	AssertRCBreak(rc);
10027	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10028	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10029
10030	/*
10031	* IA32_DEBUGCTL MSR.
10032	*/
10033	uint64_t u64Val;
10034	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10035	AssertRCBreak(rc);
10036	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10037	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10038	{
10039	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10040	}
10041	uint64_t u64DebugCtlMsr = u64Val;
10042
10043	#ifdef VBOX_STRICT
10044	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10045	AssertRCBreak(rc);
10046	Assert(u32Val == pVmcsInfo->u32EntryCtls);
10047	#endif
10048	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10049
10050	/*
10051	* RIP and RFLAGS.
10052	*/
10053	uint32_t u32Eflags;
10054	#if HC_ARCH_BITS == 64
10055	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10056	AssertRCBreak(rc);
10057	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10058	if ( !fLongModeGuest
10059	\|\| !pCtx->cs.Attr.n.u1Long)
10060	{
10061	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10062	}
10063	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10064	* must be identical if the "IA-32e mode guest" VM-entry
10065	* control is 1 and CS.L is 1. No check applies if the
10066	* CPU supports 64 linear-address bits. */
10067
10068	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10069	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10070	AssertRCBreak(rc);
10071	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10072	VMX_IGS_RFLAGS_RESERVED);
10073	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10074	u32Eflags = u64Val;
10075	#else
10076	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10077	AssertRCBreak(rc);
10078	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10079	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10080	#endif
10081
10082	if ( fLongModeGuest
10083	\|\| ( fUnrestrictedGuest
10084	&& !(u32GuestCr0 & X86_CR0_PE)))
10085	{
10086	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10087	}
10088
10089	uint32_t u32EntryInfo;
10090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10091	AssertRCBreak(rc);
10092	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
10093	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10094
10095	/*
10096	* 64-bit checks.
10097	*/
10098	#if HC_ARCH_BITS == 64
10099	if (fLongModeGuest)
10100	{
10101	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10102	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10103	}
10104
10105	if ( !fLongModeGuest
10106	&& (u32GuestCr4 & X86_CR4_PCIDE))
10107	{
10108	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10109	}
10110
10111	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10112	* 51:32 beyond the processor's physical-address width are 0. */
10113
10114	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10115	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10116	{
10117	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10118	}
10119
10120	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10121	AssertRCBreak(rc);
10122	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10123
10124	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10125	AssertRCBreak(rc);
10126	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10127	#endif
10128
10129	/*
10130	* PERF_GLOBAL MSR.
10131	*/
10132	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10133	{
10134	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10135	AssertRCBreak(rc);
10136	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10137	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10138	}
10139
10140	/*
10141	* PAT MSR.
10142	*/
10143	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10144	{
10145	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10146	AssertRCBreak(rc);
10147	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10148	for (unsigned i = 0; i < 8; i++)
10149	{
10150	uint8_t u8Val = (u64Val & 0xff);
10151	if ( u8Val != 0 /* UC */
10152	&& u8Val != 1 /* WC */
10153	&& u8Val != 4 /* WT */
10154	&& u8Val != 5 /* WP */
10155	&& u8Val != 6 /* WB */
10156	&& u8Val != 7 /* UC- */)
10157	{
10158	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10159	}
10160	u64Val >>= 8;
10161	}
10162	}
10163
10164	/*
10165	* EFER MSR.
10166	*/
10167	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10168	{
10169	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10170	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10171	AssertRCBreak(rc);
10172	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10173	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10174	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10175	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10176	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10177	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10178	* iemVmxVmentryCheckGuestState(). */
10179	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10180	\|\| !(u32GuestCr0 & X86_CR0_PG)
10181	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10182	VMX_IGS_EFER_LMA_LME_MISMATCH);
10183	}
10184
10185	/*
10186	* Segment registers.
10187	*/
10188	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10189	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10190	if (!(u32Eflags & X86_EFL_VM))
10191	{
10192	/* CS */
10193	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10194	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10195	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10196	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10197	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10198	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10199	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10200	/* CS cannot be loaded with NULL in protected mode. */
10201	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10202	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10203	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10204	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10205	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10206	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10207	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10208	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10209	else
10210	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10211
10212	/* SS */
10213	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10214	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10215	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10216	if ( !(pCtx->cr0 & X86_CR0_PE)
10217	\|\| pCtx->cs.Attr.n.u4Type == 3)
10218	{
10219	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10220	}
10221	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10222	{
10223	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10224	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10225	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10226	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10227	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10228	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10229	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10230	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10231	}
10232
10233	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10234	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10235	{
10236	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10237	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10238	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10239	\|\| pCtx->ds.Attr.n.u4Type > 11
10240	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10241	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10242	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10243	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10244	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10245	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10246	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10247	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10248	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10249	}
10250	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10251	{
10252	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10253	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10254	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10255	\|\| pCtx->es.Attr.n.u4Type > 11
10256	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10257	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10258	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10259	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10260	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10261	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10262	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10263	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10264	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10265	}
10266	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10267	{
10268	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10269	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10270	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10271	\|\| pCtx->fs.Attr.n.u4Type > 11
10272	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10273	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10274	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10275	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10276	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10277	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10278	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10279	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10280	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10281	}
10282	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10283	{
10284	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10285	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10286	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10287	\|\| pCtx->gs.Attr.n.u4Type > 11
10288	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10289	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10290	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10291	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10292	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10293	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10294	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10295	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10296	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10297	}
10298	/* 64-bit capable CPUs. */
10299	#if HC_ARCH_BITS == 64
10300	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10301	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10302	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10303	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10304	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10305	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10306	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10307	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10308	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10309	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10310	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10311	#endif
10312	}
10313	else
10314	{
10315	/* V86 mode checks. */
10316	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10317	if (pVmcsInfo->RealMode.fRealOnV86Active)
10318	{
10319	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10320	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10321	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10322	}
10323	else
10324	{
10325	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10326	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10327	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10328	}
10329
10330	/* CS */
10331	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10332	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10333	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10334	/* SS */
10335	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10336	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10337	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10338	/* DS */
10339	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10340	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10341	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10342	/* ES */
10343	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10344	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10345	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10346	/* FS */
10347	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10348	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10349	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10350	/* GS */
10351	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10352	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10353	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10354	/* 64-bit capable CPUs. */
10355	#if HC_ARCH_BITS == 64
10356	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10357	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10358	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10359	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10360	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10361	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10362	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10363	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10364	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10365	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10366	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10367	#endif
10368	}
10369
10370	/*
10371	* TR.
10372	*/
10373	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10374	/* 64-bit capable CPUs. */
10375	#if HC_ARCH_BITS == 64
10376	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10377	#endif
10378	if (fLongModeGuest)
10379	{
10380	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10381	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10382	}
10383	else
10384	{
10385	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10386	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10387	VMX_IGS_TR_ATTR_TYPE_INVALID);
10388	}
10389	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10390	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10391	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10392	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10393	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10394	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10395	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10396	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10397
10398	/*
10399	* GDTR and IDTR.
10400	*/
10401	#if HC_ARCH_BITS == 64
10402	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10403	AssertRCBreak(rc);
10404	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10405
10406	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10407	AssertRCBreak(rc);
10408	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10409	#endif
10410
10411	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10412	AssertRCBreak(rc);
10413	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10414
10415	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10416	AssertRCBreak(rc);
10417	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10418
10419	/*
10420	* Guest Non-Register State.
10421	*/
10422	/* Activity State. */
10423	uint32_t u32ActivityState;
10424	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10425	AssertRCBreak(rc);
10426	HMVMX_CHECK_BREAK( !u32ActivityState
10427	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10428	VMX_IGS_ACTIVITY_STATE_INVALID);
10429	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10430	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10431	uint32_t u32IntrState;
10432	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10433	AssertRCBreak(rc);
10434	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10435	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10436	{
10437	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10438	}
10439
10440	/** @todo Activity state and injecting interrupts. Left as a todo since we
10441	* currently don't use activity states but ACTIVE. */
10442
10443	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10444	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10445
10446	/* Guest interruptibility-state. */
10447	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10448	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10449	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10450	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10451	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10452	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10453	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10454	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
10455	{
10456	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10457	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10458	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10459	}
10460	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10461	{
10462	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10463	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10464	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10465	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10466	}
10467	/** @todo Assumes the processor is not in SMM. */
10468	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10469	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10470	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10471	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10472	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10473	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10474	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
10475	{
10476	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10477	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10478	}
10479
10480	/* Pending debug exceptions. */
10481	#if HC_ARCH_BITS == 64
10482	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10483	AssertRCBreak(rc);
10484	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10485	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10486	u32Val = u64Val; /* For pending debug exceptions checks below. */
10487	#else
10488	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10489	AssertRCBreak(rc);
10490	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10491	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10492	#endif
10493
10494	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10495	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10496	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10497	{
10498	if ( (u32Eflags & X86_EFL_TF)
10499	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10500	{
10501	/* Bit 14 is PendingDebug.BS. */
10502	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10503	}
10504	if ( !(u32Eflags & X86_EFL_TF)
10505	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10506	{
10507	/* Bit 14 is PendingDebug.BS. */
10508	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10509	}
10510	}
10511
10512	/* VMCS link pointer. */
10513	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10514	AssertRCBreak(rc);
10515	if (u64Val != UINT64_C(0xffffffffffffffff))
10516	{
10517	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10518	/** @todo Bits beyond the processor's physical-address width MBZ. */
10519	/** @todo SMM checks. */
10520	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10521	Assert(pVmcsInfo->pvShadowVmcs);
10522	VMXVMCSREVID VmcsRevId;
10523	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10524	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10525	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10526	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10527	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10528	}
10529
10530	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10531	* not using nested paging? */
10532	if ( pVM->hm.s.fNestedPaging
10533	&& !fLongModeGuest
10534	&& CPUMIsGuestInPAEModeEx(pCtx))
10535	{
10536	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10537	AssertRCBreak(rc);
10538	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10539
10540	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10541	AssertRCBreak(rc);
10542	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10543
10544	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10545	AssertRCBreak(rc);
10546	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10547
10548	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10549	AssertRCBreak(rc);
10550	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10551	}
10552
10553	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10554	if (uError == VMX_IGS_ERROR)
10555	uError = VMX_IGS_REASON_NOT_FOUND;
10556	} while (0);
10557
10558	pVCpu->hm.s.u32HMError = uError;
10559	return uError;
10560
10561	#undef HMVMX_ERROR_BREAK
10562	#undef HMVMX_CHECK_BREAK
10563	}
10564
10565
10566	/**
10567	* Map the APIC-access page for virtualizing APIC accesses.
10568	*
10569	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10570	* this not done as part of exporting guest state, see @bugref{8721}.
10571	*
10572	* @returns VBox status code.
10573	* @param pVCpu The cross context virtual CPU structure.
10574	*/
10575	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10576	{
10577	PVM pVM = pVCpu->CTX_SUFF(pVM);
10578	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10579
10580	Assert(PDMHasApic(pVM));
10581	Assert(u64MsrApicBase);
10582
10583	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10584	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10585
10586	/* Unalias the existing mapping. */
10587	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10588	AssertRCReturn(rc, rc);
10589
10590	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10591	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10592	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10593	AssertRCReturn(rc, rc);
10594
10595	/* Update the per-VCPU cache of the APIC base MSR. */
10596	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10597	return VINF_SUCCESS;
10598	}
10599
10600
10601	/**
10602	* Worker function passed to RTMpOnSpecific() that is to be called on the target
10603	* CPU.
10604	*
10605	* @param idCpu The ID for the CPU the function is called on.
10606	* @param pvUser1 Null, not used.
10607	* @param pvUser2 Null, not used.
10608	*/
10609	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
10610	{
10611	RT_NOREF3(idCpu, pvUser1, pvUser2);
10612	VMXDispatchHostNmi();
10613	}
10614
10615
10616	/**
10617	* Dispatching an NMI on the host CPU that received it.
10618	*
10619	* @returns VBox status code.
10620	* @param pVCpu The cross context virtual CPU structure.
10621	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
10622	* executing when receiving the host NMI in VMX non-root
10623	* operation.
10624	*/
10625	static int hmR0VmxExitHostNmi(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10626	{
10627	RTCPUID const idCpu = pVmcsInfo->idHostCpu;
10628
10629	/*
10630	* We don't want to delay dispatching the NMI any more than we have to. However,
10631	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
10632	* after executing guest or nested-guest code for the following reasons:
10633	*
10634	* - We would need to perform VMREADs with interrupts disabled and is orders of
10635	* magnitude worse when we run as a guest hypervisor without VMCS shadowing
10636	* supported by the host hypervisor.
10637	*
10638	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
10639	* longer period of time just for handling an edge case like host NMIs which do
10640	* not occur nearly as frequently as other VM-exits.
10641	*
10642	* Let's cover the most likely scenario first. Check if we are on the target CPU
10643	* and dispatch the NMI right away. This should be much faster than calling into
10644	* RTMpOnSpecific() machinery.
10645	*/
10646	bool fDispatched = false;
10647	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10648	if (idCpu == RTMpCpuId())
10649	{
10650	VMXDispatchHostNmi();
10651	fDispatched = true;
10652	}
10653	ASMSetFlags(fEFlags);
10654	if (fDispatched)
10655	{
10656	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10657	return VINF_SUCCESS;
10658	}
10659
10660	/*
10661	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
10662	* there should be no race or recursion even if we are unlucky enough to be preempted
10663	* (to the target CPU) without dispatching the host NMI above.
10664	*/
10665	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
10666	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
10667	}
10668
10669
10670	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10671	/**
10672	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10673	* nested-guest using hardware-assisted VMX.
10674	*
10675	* @param pVCpu The cross context virtual CPU structure.
10676	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10677	* @param pVmcsInfoGst The guest VMCS info. object.
10678	*/
10679	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10680	{
10681	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10682	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10683	Assert(pu64MsrBitmap);
10684
10685	/*
10686	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10687	* MSR that is intercepted by the guest is also intercepted while executing the
10688	* nested-guest using hardware-assisted VMX.
10689	*
10690	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10691	* nested-guest VM-exit even if the outer guest is not intercepting some
10692	* MSRs. We cannot assume the caller has initialized the nested-guest
10693	* MSR bitmap in this case.
10694	*
10695	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10696	* each VM-entry, hence initializing it once per-VM while setting up the
10697	* nested-guest VMCS is not sufficient.
10698	*/
10699	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10700	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10701	{
10702	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10703	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10704	Assert(pu64MsrBitmapNstGst);
10705	Assert(pu64MsrBitmapGst);
10706
10707	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10708	for (uint32_t i = 0; i < cFrags; i++)
10709	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10710	}
10711	else
10712	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10713	}
10714
10715
10716	/**
10717	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10718	* hardware-assisted VMX execution of the nested-guest.
10719	*
10720	* For a guest, we don't modify these controls once we set up the VMCS and hence
10721	* this function is never called.
10722	*
10723	* For nested-guests since the guest hypervisor provides these controls on every
10724	* nested-guest VM-entry and could potentially change them everytime we need to
10725	* merge them before every nested-guest VM-entry.
10726	*
10727	* @returns VBox status code.
10728	* @param pVCpu The cross context virtual CPU structure.
10729	*/
10730	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10731	{
10732	PVM pVM = pVCpu->CTX_SUFF(pVM);
10733	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10734	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10735	Assert(pVmcsNstGst);
10736
10737	/*
10738	* Merge the controls with the requirements of the guest VMCS.
10739	*
10740	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10741	* VMCS with the features supported by the physical CPU as it's already done by the
10742	* VMLAUNCH/VMRESUME instruction emulation.
10743	*
10744	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10745	* derived from the VMX features supported by the physical CPU.
10746	*/
10747
10748	/* Pin-based VM-execution controls. */
10749	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10750
10751	/* Processor-based VM-execution controls. */
10752	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10753	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10754	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10755	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10756	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10757
10758	/* Secondary processor-based VM-execution controls. */
10759	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10760	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10761	\| VMX_PROC_CTLS2_INVPCID
10762	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10763	\| VMX_PROC_CTLS2_RDTSCP
10764	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10765	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10766	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10767	\| VMX_PROC_CTLS2_VMFUNC));
10768
10769	/*
10770	* VM-entry controls:
10771	* These controls contains state that depends on the nested-guest state (primarily
10772	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10773	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10774	* properly continue executing the nested-guest if the EFER MSR changes but does not
10775	* cause a nested-guest VM-exits.
10776	*
10777	* VM-exit controls:
10778	* These controls specify the host state on return. We cannot use the controls from
10779	* the guest hypervisor state as is as it would contain the guest state rather than
10780	* the host state. Since the host state is subject to change (e.g. preemption, trips
10781	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10782	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10783	*
10784	* VM-entry MSR-load:
10785	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10786	* context by the VMLAUNCH/VMRESUME instruction emulation.
10787	*
10788	* VM-exit MSR-store:
10789	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10790	* back into the VM-exit MSR-store area.
10791	*
10792	* VM-exit MSR-load areas:
10793	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10794	* can entirely ignore what the guest hypervisor wants to load here.
10795	*/
10796
10797	/*
10798	* Exception bitmap.
10799	*
10800	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10801	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10802	* code more flexible if intercepting exceptions become more dynamic in the future we do
10803	* it as part of exporting the nested-guest state.
10804	*/
10805	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10806
10807	/*
10808	* CR0/CR4 guest/host mask.
10809	*
10810	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10811	* cause VM-exits, so we need to merge them here.
10812	*/
10813	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10814	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10815
10816	/*
10817	* Page-fault error-code mask and match.
10818	*
10819	* Although we require unrestricted guest execution (and thereby nested-paging) for
10820	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10821	* normally intercept #PFs, it might intercept them for debugging purposes.
10822	*
10823	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10824	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10825	*/
10826	uint32_t u32XcptPFMask;
10827	uint32_t u32XcptPFMatch;
10828	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10829	{
10830	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10831	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10832	}
10833	else
10834	{
10835	u32XcptPFMask = 0;
10836	u32XcptPFMatch = 0;
10837	}
10838
10839	/*
10840	* Pause-Loop exiting.
10841	*/
10842	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10843	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10844
10845	/*
10846	* I/O Bitmap.
10847	*
10848	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10849	* intercept all I/O port accesses.
10850	*/
10851	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10852
10853	/*
10854	* VMCS shadowing.
10855	*
10856	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10857	* enabled while executing the nested-guest.
10858	*/
10859	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10860
10861	/*
10862	* APIC-access page.
10863	*
10864	* The APIC-access page address has already been initialized while setting up the
10865	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it should
10866	* not be of any consequence to the host or to the guest for that matter, but we only
10867	* accept valid addresses verified by the VMLAUNCH/VMRESUME instruction emulation to keep
10868	* it simple.
10869	*/
10870
10871	/*
10872	* Virtual-APIC page and TPR threshold.
10873	*
10874	* The virtual-APIC page has already been allocated (by CPUM during VM startup) and cached
10875	* from guest memory as part of VMLAUNCH/VMRESUME instruction emulation. The host physical
10876	* address has also been updated in the nested-guest VMCS.
10877	*/
10878	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10879	RTHCPHYS HCPhysVirtApic;
10880	uint32_t u32TprThreshold;
10881	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10882	{
10883	HCPhysVirtApic = pVmcsInfoNstGst->HCPhysVirtApic;
10884	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10885	}
10886	else
10887	{
10888	HCPhysVirtApic = 0;
10889	u32TprThreshold = 0;
10890
10891	/*
10892	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10893	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10894	* be taken care of by EPT/shadow paging.
10895	*/
10896	if (pVM->hm.s.fAllow64BitGuests)
10897	{
10898	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10899	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10900	}
10901	}
10902
10903	/*
10904	* Validate basic assumptions.
10905	*/
10906	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10907	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10908	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10909
10910	/*
10911	* Commit it to the nested-guest VMCS.
10912	*/
10913	int rc = VINF_SUCCESS;
10914	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10915	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10916	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10917	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10918	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10919	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10920	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10921	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10922	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10923	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10924	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10925	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10926	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10927	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10928	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10929	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10930	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10931	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10932	{
10933	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10934	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10935	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10936	}
10937	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10938	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVmcsInfoNstGst->HCPhysVirtApic);
10939	AssertRCReturn(rc, rc);
10940
10941	/*
10942	* Update the nested-guest VMCS cache.
10943	*/
10944	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10945	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10946	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10947	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10948	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10949	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10950	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10951	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10952	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10953
10954	/*
10955	* MSR bitmap.
10956	*
10957	* The MSR bitmap address has already been initialized while setting up the nested-guest
10958	* VMCS, here we need to merge the MSR bitmaps.
10959	*/
10960	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10961	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10962
10963	return VINF_SUCCESS;
10964	}
10965	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10966
10967
10968	/**
10969	* Does the preparations before executing guest code in VT-x.
10970	*
10971	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10972	* recompiler/IEM. We must be cautious what we do here regarding committing
10973	* guest-state information into the VMCS assuming we assuredly execute the
10974	* guest in VT-x mode.
10975	*
10976	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10977	* the common-state (TRPM/forceflags), we must undo those changes so that the
10978	* recompiler/IEM can (and should) use them when it resumes guest execution.
10979	* Otherwise such operations must be done when we can no longer exit to ring-3.
10980	*
10981	* @returns Strict VBox status code (i.e. informational status codes too).
10982	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10983	* have been disabled.
10984	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10985	* pending events).
10986	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10987	* double-fault into the guest.
10988	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10989	* dispatched directly.
10990	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10991	*
10992	* @param pVCpu The cross context virtual CPU structure.
10993	* @param pVmxTransient The VMX-transient structure.
10994	* @param fStepping Whether we are single-stepping the guest in the
10995	* hypervisor debugger. Makes us ignore some of the reasons
10996	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10997	* if event dispatching took place.
10998	*/
10999	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
11000	{
11001	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11002
11003	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
11004	if (pVmxTransient->fIsNestedGuest)
11005	{
11006	RT_NOREF2(pVCpu, fStepping);
11007	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
11008	return VINF_EM_RESCHEDULE_REM;
11009	}
11010	#endif
11011
11012	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
11013	PGMRZDynMapFlushAutoSet(pVCpu);
11014	#endif
11015
11016	/*
11017	* Check and process force flag actions, some of which might require us to go back to ring-3.
11018	*/
11019	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11020	if (rcStrict == VINF_SUCCESS)
11021	{ /* FFs don't get set all the time. */ }
11022	else
11023	return rcStrict;
11024
11025	/*
11026	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
11027	*/
11028	/** @todo Doing this from ring-3 after VM setup phase causes a
11029	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
11030	* idea why atm. */
11031	PVM pVM = pVCpu->CTX_SUFF(pVM);
11032	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11033	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11034	&& PDMHasApic(pVM))
11035	{
11036	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11037	AssertRCReturn(rc, rc);
11038	}
11039
11040	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11041	/*
11042	* Merge guest VMCS controls with the nested-guest VMCS controls.
11043	*
11044	* Even if we have not executed the guest prior to this (e.g. when resuming from a
11045	* saved state), we should be okay with merging controls as we initialize the
11046	* guest VMCS controls as part of VM setup phase.
11047	*/
11048	if ( pVmxTransient->fIsNestedGuest
11049	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
11050	{
11051	int rc = hmR0VmxMergeVmcsNested(pVCpu);
11052	AssertRCReturn(rc, rc);
11053	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11054	}
11055	#endif
11056
11057	/*
11058	* Evaluate events to be injected into the guest.
11059	*
11060	* Events in TRPM can be injected without inspecting the guest state.
11061	* If any new events (interrupts/NMI) are pending currently, we try to set up the
11062	* guest to cause a VM-exit the next time they are ready to receive the event.
11063	*
11064	* With nested-guests, evaluating pending events may cause VM-exits.
11065	*/
11066	if (TRPMHasTrap(pVCpu))
11067	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11068
11069	uint32_t fIntrState;
11070	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11071
11072	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11073	/*
11074	* While evaluating pending events if something failed (unlikely) or if we were
11075	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11076	*/
11077	if (rcStrict != VINF_SUCCESS)
11078	return rcStrict;
11079	if ( pVmxTransient->fIsNestedGuest
11080	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11081	{
11082	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11083	return VINF_VMX_VMEXIT;
11084	}
11085	#else
11086	Assert(rcStrict == VINF_SUCCESS);
11087	#endif
11088
11089	/*
11090	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11091	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11092	* also result in triple-faulting the VM.
11093	*
11094	* With nested-guests, the above does not apply since unrestricted guest execution is a
11095	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
11096	*/
11097	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11098	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11099	{ /* likely */ }
11100	else
11101	{
11102	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11103	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11104	return rcStrict;
11105	}
11106
11107	/*
11108	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
11109	* import CR3 themselves. We will need to update them here, as even as late as the above
11110	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11111	* the below force flags to be set.
11112	*/
11113	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11114	{
11115	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11116	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11117	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
11118	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11119	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11120	}
11121	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11122	{
11123	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11124	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11125	}
11126
11127	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11128	/* Paranoia. */
11129	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11130	#endif
11131
11132	/*
11133	* No longjmps to ring-3 from this point on!!!
11134	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11135	* This also disables flushing of the R0-logger instance (if any).
11136	*/
11137	VMMRZCallRing3Disable(pVCpu);
11138
11139	/*
11140	* Export the guest state bits.
11141	*
11142	* We cannot perform longjmps while loading the guest state because we do not preserve the
11143	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
11144	* CPU migration.
11145	*
11146	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11147	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
11148	*/
11149	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11150	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11151	{ /* likely */ }
11152	else
11153	{
11154	VMMRZCallRing3Enable(pVCpu);
11155	return rcStrict;
11156	}
11157
11158	/*
11159	* We disable interrupts so that we don't miss any interrupts that would flag preemption
11160	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11161	* preemption disabled for a while. Since this is purely to aid the
11162	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11163	* disable interrupt on NT.
11164	*
11165	* We need to check for force-flags that could've possible been altered since we last
11166	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11167	* see @bugref{6398}).
11168	*
11169	* We also check a couple of other force-flags as a last opportunity to get the EMT back
11170	* to ring-3 before executing guest code.
11171	*/
11172	pVmxTransient->fEFlags = ASMIntDisableFlags();
11173
11174	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
11175	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11176	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11177	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
11178	{
11179	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11180	{
11181	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11182	/*
11183	* If we are executing a nested-guest make sure that we should intercept subsequent
11184	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
11185	* the VM-exit instruction emulation happy.
11186	*/
11187	if (pVmxTransient->fIsNestedGuest)
11188	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
11189	#endif
11190
11191	/*
11192	* We've injected any pending events. This is really the point of no return (to ring-3).
11193	*
11194	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
11195	* returns from this function, so do -not- enable them here.
11196	*/
11197	pVCpu->hm.s.Event.fPending = false;
11198	return VINF_SUCCESS;
11199	}
11200
11201	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11202	rcStrict = VINF_EM_RAW_INTERRUPT;
11203	}
11204	else
11205	{
11206	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11207	rcStrict = VINF_EM_RAW_TO_R3;
11208	}
11209
11210	ASMSetFlags(pVmxTransient->fEFlags);
11211	VMMRZCallRing3Enable(pVCpu);
11212
11213	return rcStrict;
11214	}
11215
11216
11217	/**
11218	* Final preparations before executing guest code using hardware-assisted VMX.
11219	*
11220	* We can no longer get preempted to a different host CPU and there are no returns
11221	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11222	* failures), this function is not intended to fail sans unrecoverable hardware
11223	* errors.
11224	*
11225	* @param pVCpu The cross context virtual CPU structure.
11226	* @param pVmxTransient The VMX-transient structure.
11227	*
11228	* @remarks Called with preemption disabled.
11229	* @remarks No-long-jump zone!!!
11230	*/
11231	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11232	{
11233	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11234	Assert(VMMR0IsLogFlushDisabled(pVCpu));
11235	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11236	Assert(!pVCpu->hm.s.Event.fPending);
11237
11238	/*
11239	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11240	*/
11241	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11242	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11243
11244	PVM pVM = pVCpu->CTX_SUFF(pVM);
11245	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11246	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11247	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11248
11249	if (!CPUMIsGuestFPUStateActive(pVCpu))
11250	{
11251	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11252	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11253	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
11254	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11255	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11256	}
11257
11258	/*
11259	* Re-export the host state bits as we may've been preempted (only happens when
11260	* thread-context hooks are used or when the VM start function changes) or if
11261	* the host CR0 is modified while loading the guest FPU state above.
11262	*
11263	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11264	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
11265	* see @bugref{8432}.
11266	*
11267	* This may also happen when switching to/from a nested-guest VMCS without leaving
11268	* ring-0.
11269	*/
11270	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11271	{
11272	hmR0VmxExportHostState(pVCpu);
11273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
11274	}
11275	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11276
11277	/*
11278	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
11279	*/
11280	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11281	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11282	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11283
11284	/*
11285	* Store status of the shared guest/host debug state at the time of VM-entry.
11286	*/
11287	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11288	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11289	{
11290	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11291	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11292	}
11293	else
11294	#endif
11295	{
11296	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11297	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11298	}
11299
11300	/*
11301	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11302	* more than one conditional check. The post-run side of our code shall determine
11303	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11304	*/
11305	if (pVmcsInfo->pbVirtApic)
11306	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11307
11308	/*
11309	* Update the host MSRs values in the VM-exit MSR-load area.
11310	*/
11311	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11312	{
11313	if (pVmcsInfo->cExitMsrLoad > 0)
11314	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11315	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11316	}
11317
11318	/*
11319	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11320	* VMX-preemption timer based on the next virtual sync clock deadline.
11321	*/
11322	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11323	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
11324	{
11325	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11326	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11327	}
11328
11329	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
11330	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
11331	if (!fIsRdtscIntercepted)
11332	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
11333	else
11334	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
11335
11336	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11337	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11338	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11339	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11340	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
11341
11342	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11343
11344	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11345	as we're about to start executing the guest . */
11346
11347	/*
11348	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11349	*
11350	* This is done this late as updating the TSC offsetting/preemption timer above
11351	* figures out if we can skip intercepting RDTSCP by calculating the number of
11352	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
11353	*/
11354	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11355	&& !fIsRdtscIntercepted)
11356	{
11357	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11358
11359	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11360	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11361	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11362	true /* fSetReadWrite /, true / fUpdateHostMsr */);
11363	AssertRC(rc);
11364	Assert(!pVmxTransient->fRemoveTscAuxMsr);
11365	pVmxTransient->fRemoveTscAuxMsr = true;
11366	}
11367
11368	#ifdef VBOX_STRICT
11369	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11370	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11371	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11372	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
11373	#endif
11374
11375	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11376	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11377	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11378	* see @bugref{9180#c54}. */
11379	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11380	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11381	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11382	#endif
11383	}
11384
11385
11386	/**
11387	* First C routine invoked after running guest code using hardware-assisted VMX.
11388	*
11389	* @param pVCpu The cross context virtual CPU structure.
11390	* @param pVmxTransient The VMX-transient structure.
11391	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11392	*
11393	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
11394	*
11395	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
11396	* unconditionally when it is safe to do so.
11397	*/
11398	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11399	{
11400	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11401
11402	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11403	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11404	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11405	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11406	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11407	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11408
11409	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11410	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11411	{
11412	uint64_t uGstTsc;
11413	if (!pVmxTransient->fIsNestedGuest)
11414	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11415	else
11416	{
11417	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11418	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11419	}
11420	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11421	}
11422
11423	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11424	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11425	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11426
11427	#if HC_ARCH_BITS == 64
11428	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11429	#endif
11430	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11431	/* The 64-on-32 switcher maintains VMCS-launch state on its own
11432	and we need to leave it alone here. */
11433	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11434	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11435	#else
11436	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11437	#endif
11438	#ifdef VBOX_STRICT
11439	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11440	#endif
11441	Assert(!ASMIntAreEnabled());
11442	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11443	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11444
11445	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
11446	/*
11447	* Clean all the VMCS fields in the transient structure before reading
11448	* anything from the VMCS.
11449	*/
11450	pVmxTransient->uExitReason = 0;
11451	pVmxTransient->uExitIntErrorCode = 0;
11452	pVmxTransient->uExitQual = 0;
11453	pVmxTransient->uGuestLinearAddr = 0;
11454	pVmxTransient->uExitIntInfo = 0;
11455	pVmxTransient->cbInstr = 0;
11456	pVmxTransient->ExitInstrInfo.u = 0;
11457	pVmxTransient->uEntryIntInfo = 0;
11458	pVmxTransient->uEntryXcptErrorCode = 0;
11459	pVmxTransient->cbEntryInstr = 0;
11460	pVmxTransient->uIdtVectoringInfo = 0;
11461	pVmxTransient->uIdtVectoringErrorCode = 0;
11462	#endif
11463
11464	/*
11465	* Save the basic VM-exit reason and check if the VM-entry failed.
11466	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11467	*/
11468	uint32_t uExitReason;
11469	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11470	AssertRC(rc);
11471	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11472	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11473
11474	/*
11475	* Log the VM-exit before logging anything else as otherwise it might be a
11476	* tad confusing what happens before and after the world-switch.
11477	*/
11478	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11479
11480	/*
11481	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11482	* bitmap permissions, if it was added before VM-entry.
11483	*/
11484	if (pVmxTransient->fRemoveTscAuxMsr)
11485	{
11486	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11487	pVmxTransient->fRemoveTscAuxMsr = false;
11488	}
11489
11490	/*
11491	* Check if VMLAUNCH/VMRESUME succeeded.
11492	* If this failed, we cause a guru meditation and cease further execution.
11493	*
11494	* However, if we are executing a nested-guest we might fail if we use the
11495	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11496	*/
11497	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11498	{
11499	/*
11500	* Update the VM-exit history array here even if the VM-entry failed due to:
11501	* - Invalid guest state.
11502	* - MSR loading.
11503	* - Machine-check event.
11504	*
11505	* In any of the above cases we will still have a "valid" VM-exit reason
11506	* despite @a fVMEntryFailed being false.
11507	*
11508	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11509	*
11510	* Note! We don't have CS or RIP at this point. Will probably address that later
11511	* by amending the history entry added here.
11512	*/
11513	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11514	UINT64_MAX, uHostTsc);
11515
11516	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11517	{
11518	VMMRZCallRing3Enable(pVCpu);
11519
11520	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11521	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11522
11523	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11524	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11525	AssertRC(rc);
11526	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11527	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11528	AssertRC(rc);
11529	#else
11530	/*
11531	* Import the guest-interruptibility state always as we need it while evaluating
11532	* injecting events on re-entry.
11533	*
11534	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11535	* checking for real-mode while exporting the state because all bits that cause
11536	* mode changes wrt CR0 are intercepted.
11537	*/
11538	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11539	AssertRC(rc);
11540	#endif
11541
11542	/*
11543	* Sync the TPR shadow with our APIC state.
11544	*
11545	* With nested-guests, mark the virtual-APIC page as dirty so it can be synced
11546	* when performing the nested-guest VM-exit.
11547	*/
11548	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11549	{
11550	if (!pVmxTransient->fIsNestedGuest)
11551	{
11552	Assert(pVmcsInfo->pbVirtApic);
11553	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11554	{
11555	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11556	AssertRC(rc);
11557	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11558	}
11559	}
11560	else
11561	pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtApicPageDirty = true;
11562	}
11563
11564	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11565	return;
11566	}
11567	}
11568	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11569	else if (pVmxTransient->fIsNestedGuest)
11570	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11571	#endif
11572	else
11573	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11574
11575	VMMRZCallRing3Enable(pVCpu);
11576	}
11577
11578
11579	/**
11580	* Runs the guest code using hardware-assisted VMX the normal way.
11581	*
11582	* @returns VBox status code.
11583	* @param pVCpu The cross context virtual CPU structure.
11584	* @param pcLoops Pointer to the number of executed loops.
11585	*/
11586	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11587	{
11588	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11589	Assert(pcLoops);
11590	Assert(*pcLoops <= cMaxResumeLoops);
11591	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11592
11593	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11594	/*
11595	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11596	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11597	* guest VMCS while entering the VMX ring-0 session.
11598	*/
11599	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11600	{
11601	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11602	if (RT_SUCCESS(rc))
11603	{ /* likely */ }
11604	else
11605	{
11606	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11607	return rc;
11608	}
11609	}
11610	#endif
11611
11612	VMXTRANSIENT VmxTransient;
11613	RT_ZERO(VmxTransient);
11614	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11615
11616	/* Paranoia. */
11617	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11618
11619	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11620	for (;;)
11621	{
11622	Assert(!HMR0SuspendPending());
11623	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11624	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11625
11626	/*
11627	* Preparatory work for running nested-guest code, this may force us to
11628	* return to ring-3.
11629	*
11630	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11631	*/
11632	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11633	if (rcStrict != VINF_SUCCESS)
11634	break;
11635
11636	/* Interrupts are disabled at this point! */
11637	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11638	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11639	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11640	/* Interrupts are re-enabled at this point! */
11641
11642	/*
11643	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11644	*/
11645	if (RT_SUCCESS(rcRun))
11646	{ /* very likely */ }
11647	else
11648	{
11649	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11650	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11651	return rcRun;
11652	}
11653
11654	/*
11655	* Profile the VM-exit.
11656	*/
11657	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11659	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11660	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11661	HMVMX_START_EXIT_DISPATCH_PROF();
11662
11663	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11664
11665	/*
11666	* Handle the VM-exit.
11667	*/
11668	#ifdef HMVMX_USE_FUNCTION_TABLE
11669	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11670	#else
11671	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11672	#endif
11673	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11674	if (rcStrict == VINF_SUCCESS)
11675	{
11676	if (++(*pcLoops) <= cMaxResumeLoops)
11677	continue;
11678	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11679	rcStrict = VINF_EM_RAW_INTERRUPT;
11680	}
11681	break;
11682	}
11683
11684	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11685	return rcStrict;
11686	}
11687
11688
11689	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11690	/**
11691	* Runs the nested-guest code using hardware-assisted VMX.
11692	*
11693	* @returns VBox status code.
11694	* @param pVCpu The cross context virtual CPU structure.
11695	* @param pcLoops Pointer to the number of executed loops.
11696	*
11697	* @sa hmR0VmxRunGuestCodeNormal.
11698	*/
11699	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11700	{
11701	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11702	Assert(pcLoops);
11703	Assert(*pcLoops <= cMaxResumeLoops);
11704	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11705
11706	/*
11707	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11708	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11709	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11710	*/
11711	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11712	{
11713	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11714	if (RT_SUCCESS(rc))
11715	{ /* likely */ }
11716	else
11717	{
11718	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11719	return rc;
11720	}
11721	}
11722
11723	VMXTRANSIENT VmxTransient;
11724	RT_ZERO(VmxTransient);
11725	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11726	VmxTransient.fIsNestedGuest = true;
11727
11728	/* Paranoia. */
11729	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11730
11731	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11732	for (;;)
11733	{
11734	Assert(!HMR0SuspendPending());
11735	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11736	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11737
11738	/*
11739	* Preparatory work for running guest code, this may force us to
11740	* return to ring-3.
11741	*
11742	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11743	*/
11744	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11745	if (rcStrict != VINF_SUCCESS)
11746	break;
11747
11748	/* Interrupts are disabled at this point! */
11749	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11750	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11751	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11752	/* Interrupts are re-enabled at this point! */
11753
11754	/*
11755	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11756	*/
11757	if (RT_SUCCESS(rcRun))
11758	{ /* very likely */ }
11759	else
11760	{
11761	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11762	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11763	return rcRun;
11764	}
11765
11766	/*
11767	* Profile the VM-exit.
11768	*/
11769	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11770	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11771	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11772	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11773	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11774	HMVMX_START_EXIT_DISPATCH_PROF();
11775
11776	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11777
11778	/*
11779	* Handle the VM-exit.
11780	*/
11781	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11782	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11783	if (rcStrict == VINF_SUCCESS)
11784	{
11785	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11786	{
11787	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11788	rcStrict = VINF_VMX_VMEXIT;
11789	}
11790	else
11791	{
11792	if (++(*pcLoops) <= cMaxResumeLoops)
11793	continue;
11794	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11795	rcStrict = VINF_EM_RAW_INTERRUPT;
11796	}
11797	}
11798	else
11799	Assert(rcStrict != VINF_VMX_VMEXIT);
11800	break;
11801	}
11802
11803	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11804	return rcStrict;
11805	}
11806	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11807
11808
11809	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11810	* probes.
11811	*
11812	* The following few functions and associated structure contains the bloat
11813	* necessary for providing detailed debug events and dtrace probes as well as
11814	* reliable host side single stepping. This works on the principle of
11815	* "subclassing" the normal execution loop and workers. We replace the loop
11816	* method completely and override selected helpers to add necessary adjustments
11817	* to their core operation.
11818	*
11819	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11820	* any performance for debug and analysis features.
11821	*
11822	* @{
11823	*/
11824
11825	/**
11826	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11827	* the debug run loop.
11828	*/
11829	typedef struct VMXRUNDBGSTATE
11830	{
11831	/** The RIP we started executing at. This is for detecting that we stepped. */
11832	uint64_t uRipStart;
11833	/** The CS we started executing with. */
11834	uint16_t uCsStart;
11835
11836	/** Whether we've actually modified the 1st execution control field. */
11837	bool fModifiedProcCtls : 1;
11838	/** Whether we've actually modified the 2nd execution control field. */
11839	bool fModifiedProcCtls2 : 1;
11840	/** Whether we've actually modified the exception bitmap. */
11841	bool fModifiedXcptBitmap : 1;
11842
11843	/** We desire the modified the CR0 mask to be cleared. */
11844	bool fClearCr0Mask : 1;
11845	/** We desire the modified the CR4 mask to be cleared. */
11846	bool fClearCr4Mask : 1;
11847	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11848	uint32_t fCpe1Extra;
11849	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11850	uint32_t fCpe1Unwanted;
11851	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11852	uint32_t fCpe2Extra;
11853	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11854	uint32_t bmXcptExtra;
11855	/** The sequence number of the Dtrace provider settings the state was
11856	* configured against. */
11857	uint32_t uDtraceSettingsSeqNo;
11858	/** VM-exits to check (one bit per VM-exit). */
11859	uint32_t bmExitsToCheck[3];
11860
11861	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11862	uint32_t fProcCtlsInitial;
11863	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11864	uint32_t fProcCtls2Initial;
11865	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11866	uint32_t bmXcptInitial;
11867	} VMXRUNDBGSTATE;
11868	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11869	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11870
11871
11872	/**
11873	* Initializes the VMXRUNDBGSTATE structure.
11874	*
11875	* @param pVCpu The cross context virtual CPU structure of the
11876	* calling EMT.
11877	* @param pVmxTransient The VMX-transient structure.
11878	* @param pDbgState The debug state to initialize.
11879	*/
11880	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11881	{
11882	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11883	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11884
11885	pDbgState->fModifiedProcCtls = false;
11886	pDbgState->fModifiedProcCtls2 = false;
11887	pDbgState->fModifiedXcptBitmap = false;
11888	pDbgState->fClearCr0Mask = false;
11889	pDbgState->fClearCr4Mask = false;
11890	pDbgState->fCpe1Extra = 0;
11891	pDbgState->fCpe1Unwanted = 0;
11892	pDbgState->fCpe2Extra = 0;
11893	pDbgState->bmXcptExtra = 0;
11894	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11895	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11896	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11897	}
11898
11899
11900	/**
11901	* Updates the VMSC fields with changes requested by @a pDbgState.
11902	*
11903	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11904	* immediately before executing guest code, i.e. when interrupts are disabled.
11905	* We don't check status codes here as we cannot easily assert or return in the
11906	* latter case.
11907	*
11908	* @param pVCpu The cross context virtual CPU structure.
11909	* @param pVmxTransient The VMX-transient structure.
11910	* @param pDbgState The debug state.
11911	*/
11912	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11913	{
11914	/*
11915	* Ensure desired flags in VMCS control fields are set.
11916	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11917	*
11918	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11919	* there should be no stale data in pCtx at this point.
11920	*/
11921	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11922	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11923	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11924	{
11925	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11926	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11927	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11928	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11929	pDbgState->fModifiedProcCtls = true;
11930	}
11931
11932	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11933	{
11934	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11935	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11936	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11937	pDbgState->fModifiedProcCtls2 = true;
11938	}
11939
11940	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11941	{
11942	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11943	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11944	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11945	pDbgState->fModifiedXcptBitmap = true;
11946	}
11947
11948	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11949	{
11950	pVmcsInfo->u64Cr0Mask = 0;
11951	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11952	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11953	}
11954
11955	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11956	{
11957	pVmcsInfo->u64Cr4Mask = 0;
11958	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11959	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11960	}
11961
11962	NOREF(pVCpu);
11963	}
11964
11965
11966	/**
11967	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11968	* re-entry next time around.
11969	*
11970	* @returns Strict VBox status code (i.e. informational status codes too).
11971	* @param pVCpu The cross context virtual CPU structure.
11972	* @param pVmxTransient The VMX-transient structure.
11973	* @param pDbgState The debug state.
11974	* @param rcStrict The return code from executing the guest using single
11975	* stepping.
11976	*/
11977	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11978	VBOXSTRICTRC rcStrict)
11979	{
11980	/*
11981	* Restore VM-exit control settings as we may not reenter this function the
11982	* next time around.
11983	*/
11984	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11985
11986	/* We reload the initial value, trigger what we can of recalculations the
11987	next time around. From the looks of things, that's all that's required atm. */
11988	if (pDbgState->fModifiedProcCtls)
11989	{
11990	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11991	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11992	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11993	AssertRCReturn(rc2, rc2);
11994	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11995	}
11996
11997	/* We're currently the only ones messing with this one, so just restore the
11998	cached value and reload the field. */
11999	if ( pDbgState->fModifiedProcCtls2
12000	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
12001	{
12002	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
12003	AssertRCReturn(rc2, rc2);
12004	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
12005	}
12006
12007	/* If we've modified the exception bitmap, we restore it and trigger
12008	reloading and partial recalculation the next time around. */
12009	if (pDbgState->fModifiedXcptBitmap)
12010	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
12011
12012	return rcStrict;
12013	}
12014
12015
12016	/**
12017	* Configures VM-exit controls for current DBGF and DTrace settings.
12018	*
12019	* This updates @a pDbgState and the VMCS execution control fields to reflect
12020	* the necessary VM-exits demanded by DBGF and DTrace.
12021	*
12022	* @param pVCpu The cross context virtual CPU structure.
12023	* @param pVmxTransient The VMX-transient structure. May update
12024	* fUpdatedTscOffsettingAndPreemptTimer.
12025	* @param pDbgState The debug state.
12026	*/
12027	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12028	{
12029	/*
12030	* Take down the dtrace serial number so we can spot changes.
12031	*/
12032	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
12033	ASMCompilerBarrier();
12034
12035	/*
12036	* We'll rebuild most of the middle block of data members (holding the
12037	* current settings) as we go along here, so start by clearing it all.
12038	*/
12039	pDbgState->bmXcptExtra = 0;
12040	pDbgState->fCpe1Extra = 0;
12041	pDbgState->fCpe1Unwanted = 0;
12042	pDbgState->fCpe2Extra = 0;
12043	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
12044	pDbgState->bmExitsToCheck[i] = 0;
12045
12046	/*
12047	* Software interrupts (INT XXh) - no idea how to trigger these...
12048	*/
12049	PVM pVM = pVCpu->CTX_SUFF(pVM);
12050	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
12051	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
12052	{
12053	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12054	}
12055
12056	/*
12057	* INT3 breakpoints - triggered by #BP exceptions.
12058	*/
12059	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
12060	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12061
12062	/*
12063	* Exception bitmap and XCPT events+probes.
12064	*/
12065	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
12066	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
12067	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
12068
12069	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
12070	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
12071	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12072	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
12073	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
12074	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
12075	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
12076	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
12077	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
12078	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
12079	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
12080	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
12081	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
12082	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
12083	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
12084	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
12085	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
12086	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
12087
12088	if (pDbgState->bmXcptExtra)
12089	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12090
12091	/*
12092	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12093	*
12094	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12095	* So, when adding/changing/removing please don't forget to update it.
12096	*
12097	* Some of the macros are picking up local variables to save horizontal space,
12098	* (being able to see it in a table is the lesser evil here).
12099	*/
12100	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12101	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12102	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12103	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12104	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12105	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12106	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12107	} else do { } while (0)
12108	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12109	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12110	{ \
12111	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
12112	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12113	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12114	} else do { } while (0)
12115	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12116	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12117	{ \
12118	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
12119	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12120	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12121	} else do { } while (0)
12122	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12123	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12124	{ \
12125	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
12126	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12127	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12128	} else do { } while (0)
12129
12130	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12131	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12132	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12133	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12134	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12135
12136	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12137	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12138	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12139	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12140	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12141	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12142	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12143	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12144	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12145	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12146	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12147	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12148	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12149	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12150	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12151	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12152	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12153	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12154	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12155	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12156	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12157	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12158	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12159	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12160	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12161	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12162	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12163	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12164	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12165	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12166	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12167	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12168	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12169	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12170	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12171	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12172
12173	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12174	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12175	{
12176	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
12177	\| CPUMCTX_EXTRN_APIC_TPR);
12178	AssertRC(rc);
12179
12180	#if 0 /** @todo fix me */
12181	pDbgState->fClearCr0Mask = true;
12182	pDbgState->fClearCr4Mask = true;
12183	#endif
12184	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12185	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
12186	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12187	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
12188	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12189	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12190	require clearing here and in the loop if we start using it. */
12191	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12192	}
12193	else
12194	{
12195	if (pDbgState->fClearCr0Mask)
12196	{
12197	pDbgState->fClearCr0Mask = false;
12198	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12199	}
12200	if (pDbgState->fClearCr4Mask)
12201	{
12202	pDbgState->fClearCr4Mask = false;
12203	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12204	}
12205	}
12206	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12207	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12208
12209	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12210	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12211	{
12212	/** @todo later, need to fix handler as it assumes this won't usually happen. */
12213	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12214	}
12215	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12216	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12217
12218	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12219	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12220	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12221	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12222	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12223	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12224	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12225	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12226	#if 0 /** @todo too slow, fix handler. */
12227	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12228	#endif
12229	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12230
12231	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12232	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12233	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12234	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12235	{
12236	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12237	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12238	}
12239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12240	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12242	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12243
12244	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12245	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
12246	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12247	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
12248	{
12249	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12250	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12251	}
12252	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12253	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12254	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12255	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12256
12257	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12258	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12259	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12260	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12261	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12262	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12263	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12264	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12265	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12266	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12267	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12268	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12269	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12270	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12271	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12272	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12273	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12274	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12275	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12276	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12277	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12278	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12279
12280	#undef IS_EITHER_ENABLED
12281	#undef SET_ONLY_XBM_IF_EITHER_EN
12282	#undef SET_CPE1_XBM_IF_EITHER_EN
12283	#undef SET_CPEU_XBM_IF_EITHER_EN
12284	#undef SET_CPE2_XBM_IF_EITHER_EN
12285
12286	/*
12287	* Sanitize the control stuff.
12288	*/
12289	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12290	if (pDbgState->fCpe2Extra)
12291	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12292	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12293	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12294	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12295	{
12296	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12297	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12298	}
12299
12300	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12301	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12302	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12303	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12304	}
12305
12306
12307	/**
12308	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
12309	* appropriate.
12310	*
12311	* The caller has checked the VM-exit against the
12312	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12313	* already, so we don't have to do that either.
12314	*
12315	* @returns Strict VBox status code (i.e. informational status codes too).
12316	* @param pVCpu The cross context virtual CPU structure.
12317	* @param pVmxTransient The VMX-transient structure.
12318	* @param uExitReason The VM-exit reason.
12319	*
12320	* @remarks The name of this function is displayed by dtrace, so keep it short
12321	* and to the point. No longer than 33 chars long, please.
12322	*/
12323	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12324	{
12325	/*
12326	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12327	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
12328	*
12329	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12330	* does. Must add/change/remove both places. Same ordering, please.
12331	*
12332	* Added/removed events must also be reflected in the next section
12333	* where we dispatch dtrace events.
12334	*/
12335	bool fDtrace1 = false;
12336	bool fDtrace2 = false;
12337	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12338	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12339	uint32_t uEventArg = 0;
12340	#define SET_EXIT(a_EventSubName) \
12341	do { \
12342	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12343	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12344	} while (0)
12345	#define SET_BOTH(a_EventSubName) \
12346	do { \
12347	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12348	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12349	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12350	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12351	} while (0)
12352	switch (uExitReason)
12353	{
12354	case VMX_EXIT_MTF:
12355	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12356
12357	case VMX_EXIT_XCPT_OR_NMI:
12358	{
12359	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12360	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12361	{
12362	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12363	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12364	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12365	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12366	{
12367	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12368	{
12369	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12370	uEventArg = pVmxTransient->uExitIntErrorCode;
12371	}
12372	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12373	switch (enmEvent1)
12374	{
12375	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12376	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12377	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12378	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12379	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12380	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12381	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12382	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12383	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12384	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12385	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12386	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12387	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12388	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12389	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12390	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12391	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12392	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12393	default: break;
12394	}
12395	}
12396	else
12397	AssertFailed();
12398	break;
12399
12400	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12401	uEventArg = idxVector;
12402	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12403	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12404	break;
12405	}
12406	break;
12407	}
12408
12409	case VMX_EXIT_TRIPLE_FAULT:
12410	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12411	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12412	break;
12413	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12414	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12415	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12416	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12417	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12418
12419	/* Instruction specific VM-exits: */
12420	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12421	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12422	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12423	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12424	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12425	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12426	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12427	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12428	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12429	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12430	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12431	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12432	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12433	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12434	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12435	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12436	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12437	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12438	case VMX_EXIT_MOV_CRX:
12439	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12440	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12441	SET_BOTH(CRX_READ);
12442	else
12443	SET_BOTH(CRX_WRITE);
12444	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12445	break;
12446	case VMX_EXIT_MOV_DRX:
12447	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12448	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12449	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12450	SET_BOTH(DRX_READ);
12451	else
12452	SET_BOTH(DRX_WRITE);
12453	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12454	break;
12455	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12456	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12457	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12458	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12459	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12460	case VMX_EXIT_GDTR_IDTR_ACCESS:
12461	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12462	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12463	{
12464	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12465	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12466	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12467	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12468	}
12469	break;
12470
12471	case VMX_EXIT_LDTR_TR_ACCESS:
12472	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12473	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12474	{
12475	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12476	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12477	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12478	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12479	}
12480	break;
12481
12482	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12483	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12484	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12485	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12486	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12487	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12488	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12489	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12490	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12491	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12492	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12493
12494	/* Events that aren't relevant at this point. */
12495	case VMX_EXIT_EXT_INT:
12496	case VMX_EXIT_INT_WINDOW:
12497	case VMX_EXIT_NMI_WINDOW:
12498	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12499	case VMX_EXIT_PREEMPT_TIMER:
12500	case VMX_EXIT_IO_INSTR:
12501	break;
12502
12503	/* Errors and unexpected events. */
12504	case VMX_EXIT_INIT_SIGNAL:
12505	case VMX_EXIT_SIPI:
12506	case VMX_EXIT_IO_SMI:
12507	case VMX_EXIT_SMI:
12508	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12509	case VMX_EXIT_ERR_MSR_LOAD:
12510	case VMX_EXIT_ERR_MACHINE_CHECK:
12511	case VMX_EXIT_PML_FULL:
12512	case VMX_EXIT_VIRTUALIZED_EOI:
12513	break;
12514
12515	default:
12516	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12517	break;
12518	}
12519	#undef SET_BOTH
12520	#undef SET_EXIT
12521
12522	/*
12523	* Dtrace tracepoints go first. We do them here at once so we don't
12524	* have to copy the guest state saving and stuff a few dozen times.
12525	* Down side is that we've got to repeat the switch, though this time
12526	* we use enmEvent since the probes are a subset of what DBGF does.
12527	*/
12528	if (fDtrace1 \|\| fDtrace2)
12529	{
12530	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12531	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12532	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12533	switch (enmEvent1)
12534	{
12535	/** @todo consider which extra parameters would be helpful for each probe. */
12536	case DBGFEVENT_END: break;
12537	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12538	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12539	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12540	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12541	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12542	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12543	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12544	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12545	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12546	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12547	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12548	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12549	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12550	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12551	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12552	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12553	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12554	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12555	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12556	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12557	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12558	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12559	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12560	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12561	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12562	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12563	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12564	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12565	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12566	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12567	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12568	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12569	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12570	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12571	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12572	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12573	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12574	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12575	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12576	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12577	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12578	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12579	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12580	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12581	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12582	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12583	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12584	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12585	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12586	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12587	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12588	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12589	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12590	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12591	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12592	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12593	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12594	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12595	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12596	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12597	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12598	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12599	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12600	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12601	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12602	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12603	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12604	}
12605	switch (enmEvent2)
12606	{
12607	/** @todo consider which extra parameters would be helpful for each probe. */
12608	case DBGFEVENT_END: break;
12609	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12610	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12611	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12612	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12613	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12614	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12615	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12616	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12617	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12618	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12619	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12620	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12621	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12622	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12623	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12624	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12625	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12626	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12627	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12628	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12629	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12630	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12631	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12632	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12633	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12634	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12635	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12636	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12637	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12638	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12639	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12640	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12641	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12642	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12643	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12644	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12645	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12646	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12647	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12648	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12649	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12650	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12651	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12652	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12653	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12654	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12655	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12656	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12657	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12658	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12659	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12660	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12661	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12662	}
12663	}
12664
12665	/*
12666	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12667	* the DBGF call will do a full check).
12668	*
12669	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12670	* Note! If we have to events, we prioritize the first, i.e. the instruction
12671	* one, in order to avoid event nesting.
12672	*/
12673	PVM pVM = pVCpu->CTX_SUFF(pVM);
12674	if ( enmEvent1 != DBGFEVENT_END
12675	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12676	{
12677	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12678	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12679	if (rcStrict != VINF_SUCCESS)
12680	return rcStrict;
12681	}
12682	else if ( enmEvent2 != DBGFEVENT_END
12683	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12684	{
12685	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12686	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12687	if (rcStrict != VINF_SUCCESS)
12688	return rcStrict;
12689	}
12690
12691	return VINF_SUCCESS;
12692	}
12693
12694
12695	/**
12696	* Single-stepping VM-exit filtering.
12697	*
12698	* This is preprocessing the VM-exits and deciding whether we've gotten far
12699	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12700	* handling is performed.
12701	*
12702	* @returns Strict VBox status code (i.e. informational status codes too).
12703	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12704	* @param pVmxTransient The VMX-transient structure.
12705	* @param pDbgState The debug state.
12706	*/
12707	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12708	{
12709	/*
12710	* Expensive (saves context) generic dtrace VM-exit probe.
12711	*/
12712	uint32_t const uExitReason = pVmxTransient->uExitReason;
12713	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12714	{ /* more likely */ }
12715	else
12716	{
12717	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12718	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12719	AssertRC(rc);
12720	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12721	}
12722
12723	/*
12724	* Check for host NMI, just to get that out of the way.
12725	*/
12726	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12727	{ /* normally likely */ }
12728	else
12729	{
12730	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12731	AssertRCReturn(rc2, rc2);
12732	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12733	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12734	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12735	}
12736
12737	/*
12738	* Check for single stepping event if we're stepping.
12739	*/
12740	if (pVCpu->hm.s.fSingleInstruction)
12741	{
12742	switch (uExitReason)
12743	{
12744	case VMX_EXIT_MTF:
12745	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12746
12747	/* Various events: */
12748	case VMX_EXIT_XCPT_OR_NMI:
12749	case VMX_EXIT_EXT_INT:
12750	case VMX_EXIT_TRIPLE_FAULT:
12751	case VMX_EXIT_INT_WINDOW:
12752	case VMX_EXIT_NMI_WINDOW:
12753	case VMX_EXIT_TASK_SWITCH:
12754	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12755	case VMX_EXIT_APIC_ACCESS:
12756	case VMX_EXIT_EPT_VIOLATION:
12757	case VMX_EXIT_EPT_MISCONFIG:
12758	case VMX_EXIT_PREEMPT_TIMER:
12759
12760	/* Instruction specific VM-exits: */
12761	case VMX_EXIT_CPUID:
12762	case VMX_EXIT_GETSEC:
12763	case VMX_EXIT_HLT:
12764	case VMX_EXIT_INVD:
12765	case VMX_EXIT_INVLPG:
12766	case VMX_EXIT_RDPMC:
12767	case VMX_EXIT_RDTSC:
12768	case VMX_EXIT_RSM:
12769	case VMX_EXIT_VMCALL:
12770	case VMX_EXIT_VMCLEAR:
12771	case VMX_EXIT_VMLAUNCH:
12772	case VMX_EXIT_VMPTRLD:
12773	case VMX_EXIT_VMPTRST:
12774	case VMX_EXIT_VMREAD:
12775	case VMX_EXIT_VMRESUME:
12776	case VMX_EXIT_VMWRITE:
12777	case VMX_EXIT_VMXOFF:
12778	case VMX_EXIT_VMXON:
12779	case VMX_EXIT_MOV_CRX:
12780	case VMX_EXIT_MOV_DRX:
12781	case VMX_EXIT_IO_INSTR:
12782	case VMX_EXIT_RDMSR:
12783	case VMX_EXIT_WRMSR:
12784	case VMX_EXIT_MWAIT:
12785	case VMX_EXIT_MONITOR:
12786	case VMX_EXIT_PAUSE:
12787	case VMX_EXIT_GDTR_IDTR_ACCESS:
12788	case VMX_EXIT_LDTR_TR_ACCESS:
12789	case VMX_EXIT_INVEPT:
12790	case VMX_EXIT_RDTSCP:
12791	case VMX_EXIT_INVVPID:
12792	case VMX_EXIT_WBINVD:
12793	case VMX_EXIT_XSETBV:
12794	case VMX_EXIT_RDRAND:
12795	case VMX_EXIT_INVPCID:
12796	case VMX_EXIT_VMFUNC:
12797	case VMX_EXIT_RDSEED:
12798	case VMX_EXIT_XSAVES:
12799	case VMX_EXIT_XRSTORS:
12800	{
12801	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12802	AssertRCReturn(rc, rc);
12803	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12804	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12805	return VINF_EM_DBG_STEPPED;
12806	break;
12807	}
12808
12809	/* Errors and unexpected events: */
12810	case VMX_EXIT_INIT_SIGNAL:
12811	case VMX_EXIT_SIPI:
12812	case VMX_EXIT_IO_SMI:
12813	case VMX_EXIT_SMI:
12814	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12815	case VMX_EXIT_ERR_MSR_LOAD:
12816	case VMX_EXIT_ERR_MACHINE_CHECK:
12817	case VMX_EXIT_PML_FULL:
12818	case VMX_EXIT_VIRTUALIZED_EOI:
12819	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12820	break;
12821
12822	default:
12823	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12824	break;
12825	}
12826	}
12827
12828	/*
12829	* Check for debugger event breakpoints and dtrace probes.
12830	*/
12831	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12832	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12833	{
12834	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12835	if (rcStrict != VINF_SUCCESS)
12836	return rcStrict;
12837	}
12838
12839	/*
12840	* Normal processing.
12841	*/
12842	#ifdef HMVMX_USE_FUNCTION_TABLE
12843	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12844	#else
12845	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12846	#endif
12847	}
12848
12849
12850	/**
12851	* Single steps guest code using hardware-assisted VMX.
12852	*
12853	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12854	* but single-stepping through the hypervisor debugger.
12855	*
12856	* @returns Strict VBox status code (i.e. informational status codes too).
12857	* @param pVCpu The cross context virtual CPU structure.
12858	* @param pcLoops Pointer to the number of executed loops.
12859	*
12860	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12861	*/
12862	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12863	{
12864	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12865	Assert(pcLoops);
12866	Assert(*pcLoops <= cMaxResumeLoops);
12867
12868	VMXTRANSIENT VmxTransient;
12869	RT_ZERO(VmxTransient);
12870	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12871
12872	/* Set HMCPU indicators. */
12873	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12874	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12875	pVCpu->hm.s.fDebugWantRdTscExit = false;
12876	pVCpu->hm.s.fUsingDebugLoop = true;
12877
12878	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12879	VMXRUNDBGSTATE DbgState;
12880	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12881	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12882
12883	/*
12884	* The loop.
12885	*/
12886	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12887	for (;;)
12888	{
12889	Assert(!HMR0SuspendPending());
12890	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12891	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12892	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12893
12894	/* Set up VM-execution controls the next two can respond to. */
12895	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12896
12897	/*
12898	* Preparatory work for running guest code, this may force us to
12899	* return to ring-3.
12900	*
12901	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12902	*/
12903	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12904	if (rcStrict != VINF_SUCCESS)
12905	break;
12906
12907	/* Interrupts are disabled at this point! */
12908	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12909
12910	/* Override any obnoxious code in the above two calls. */
12911	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12912
12913	/*
12914	* Finally execute the guest.
12915	*/
12916	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12917
12918	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12919	/* Interrupts are re-enabled at this point! */
12920
12921	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12922	if (RT_SUCCESS(rcRun))
12923	{ /* very likely */ }
12924	else
12925	{
12926	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12927	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12928	return rcRun;
12929	}
12930
12931	/* Profile the VM-exit. */
12932	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12934	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12935	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12936	HMVMX_START_EXIT_DISPATCH_PROF();
12937
12938	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12939
12940	/*
12941	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12942	*/
12943	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12944	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12945	if (rcStrict != VINF_SUCCESS)
12946	break;
12947	if (++(*pcLoops) > cMaxResumeLoops)
12948	{
12949	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12950	rcStrict = VINF_EM_RAW_INTERRUPT;
12951	break;
12952	}
12953
12954	/*
12955	* Stepping: Did the RIP change, if so, consider it a single step.
12956	* Otherwise, make sure one of the TFs gets set.
12957	*/
12958	if (fStepping)
12959	{
12960	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12961	AssertRC(rc);
12962	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12963	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12964	{
12965	rcStrict = VINF_EM_DBG_STEPPED;
12966	break;
12967	}
12968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12969	}
12970
12971	/*
12972	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12973	*/
12974	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12975	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12976	}
12977
12978	/*
12979	* Clear the X86_EFL_TF if necessary.
12980	*/
12981	if (pVCpu->hm.s.fClearTrapFlag)
12982	{
12983	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12984	AssertRC(rc);
12985	pVCpu->hm.s.fClearTrapFlag = false;
12986	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12987	}
12988	/** @todo there seems to be issues with the resume flag when the monitor trap
12989	* flag is pending without being used. Seen early in bios init when
12990	* accessing APIC page in protected mode. */
12991
12992	/*
12993	* Restore VM-exit control settings as we may not re-enter this function the
12994	* next time around.
12995	*/
12996	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12997
12998	/* Restore HMCPU indicators. */
12999	pVCpu->hm.s.fUsingDebugLoop = false;
13000	pVCpu->hm.s.fDebugWantRdTscExit = false;
13001	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
13002
13003	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
13004	return rcStrict;
13005	}
13006
13007
13008	/** @} */
13009
13010
13011	/**
13012	* Checks if any expensive dtrace probes are enabled and we should go to the
13013	* debug loop.
13014	*
13015	* @returns true if we should use debug loop, false if not.
13016	*/
13017	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
13018	{
13019	/* It's probably faster to OR the raw 32-bit counter variables together.
13020	Since the variables are in an array and the probes are next to one
13021	another (more or less), we have good locality. So, better read
13022	eight-nine cache lines ever time and only have one conditional, than
13023	128+ conditionals, right? */
13024	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
13025	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
13026	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
13027	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
13028	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
13029	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
13030	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
13031	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
13032	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
13033	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
13034	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
13035	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
13036	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
13037	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
13038	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
13039	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
13040	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
13041	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
13042	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
13043	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
13044	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
13045	) != 0
13046	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
13047	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
13048	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
13049	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
13050	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
13051	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
13052	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
13053	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
13054	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
13055	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
13056	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
13057	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
13058	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
13059	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
13060	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
13061	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
13062	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
13063	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
13064	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
13065	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
13066	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
13067	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
13068	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
13069	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
13070	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
13071	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
13072	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
13073	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
13074	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
13075	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13076	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13077	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13078	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13079	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13080	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13081	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13082	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13083	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13084	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13085	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13086	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13087	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13088	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13089	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13090	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13091	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13092	) != 0
13093	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13094	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
13095	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13096	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13097	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13098	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
13099	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13100	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13101	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13102	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13103	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13104	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13105	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13106	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13107	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13108	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13109	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13110	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13111	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13112	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13113	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13114	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13115	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13116	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13117	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13118	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
13119	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
13120	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13121	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
13122	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13123	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13124	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13125	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13126	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13127	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13128	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13129	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13130	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13131	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13132	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13133	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13134	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13135	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13136	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13137	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13138	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13139	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13140	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13141	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13142	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13143	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13144	) != 0;
13145	}
13146
13147
13148	/**
13149	* Runs the guest using hardware-assisted VMX.
13150	*
13151	* @returns Strict VBox status code (i.e. informational status codes too).
13152	* @param pVCpu The cross context virtual CPU structure.
13153	*/
13154	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13155	{
13156	AssertPtr(pVCpu);
13157	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13158	Assert(VMMRZCallRing3IsEnabled(pVCpu));
13159	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13160	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13161
13162	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13163
13164	VBOXSTRICTRC rcStrict;
13165	uint32_t cLoops = 0;
13166	for (;;)
13167	{
13168	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13169	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13170	#else
13171	bool const fInNestedGuestMode = false;
13172	#endif
13173	if (!fInNestedGuestMode)
13174	{
13175	if ( !pVCpu->hm.s.fUseDebugLoop
13176	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
13177	&& !DBGFIsStepping(pVCpu)
13178	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13179	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13180	else
13181	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13182	}
13183	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13184	else
13185	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13186
13187	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13188	{
13189	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
13190	continue;
13191	}
13192	if (rcStrict == VINF_VMX_VMEXIT)
13193	{
13194	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
13195	continue;
13196	}
13197	#endif
13198	break;
13199	}
13200
13201	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13202	switch (rcLoop)
13203	{
13204	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13205	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13206	}
13207
13208	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13209	if (RT_FAILURE(rc2))
13210	{
13211	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13212	rcStrict = rc2;
13213	}
13214	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13215	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13216	return rcStrict;
13217	}
13218
13219
13220	#ifndef HMVMX_USE_FUNCTION_TABLE
13221	/**
13222	* Handles a guest VM-exit from hardware-assisted VMX execution.
13223	*
13224	* @returns Strict VBox status code (i.e. informational status codes too).
13225	* @param pVCpu The cross context virtual CPU structure.
13226	* @param pVmxTransient The VMX-transient structure.
13227	*/
13228	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13229	{
13230	#ifdef DEBUG_ramshankar
13231	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13232	do { \
13233	if (a_fSave != 0) \
13234	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
13235	VBOXSTRICTRC rcStrict = a_CallExpr; \
13236	if (a_fSave != 0) \
13237	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13238	return rcStrict; \
13239	} while (0)
13240	#else
13241	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13242	#endif
13243	uint32_t const uExitReason = pVmxTransient->uExitReason;
13244	switch (uExitReason)
13245	{
13246	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13247	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13248	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13249	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13250	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13251	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13252	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13253	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13254	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13255	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13256	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13257	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13258	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13259	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13260	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13261	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13262	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13263	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13264	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13265	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13266	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13267	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13268	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13269	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13270	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13271	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13272	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13273	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13274	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13275	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13276	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13277	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13278	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13279	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13280	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13281	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13282	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13283	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13284	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13285	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13286	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13287	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13288	#else
13289	case VMX_EXIT_VMCLEAR:
13290	case VMX_EXIT_VMLAUNCH:
13291	case VMX_EXIT_VMPTRLD:
13292	case VMX_EXIT_VMPTRST:
13293	case VMX_EXIT_VMREAD:
13294	case VMX_EXIT_VMRESUME:
13295	case VMX_EXIT_VMWRITE:
13296	case VMX_EXIT_VMXOFF:
13297	case VMX_EXIT_VMXON:
13298	case VMX_EXIT_INVVPID:
13299	case VMX_EXIT_INVEPT:
13300	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13301	#endif
13302
13303	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13304	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13305	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13306
13307	case VMX_EXIT_INIT_SIGNAL:
13308	case VMX_EXIT_SIPI:
13309	case VMX_EXIT_IO_SMI:
13310	case VMX_EXIT_SMI:
13311	case VMX_EXIT_ERR_MSR_LOAD:
13312	case VMX_EXIT_ERR_MACHINE_CHECK:
13313	case VMX_EXIT_PML_FULL:
13314	case VMX_EXIT_VIRTUALIZED_EOI:
13315	case VMX_EXIT_GDTR_IDTR_ACCESS:
13316	case VMX_EXIT_LDTR_TR_ACCESS:
13317	case VMX_EXIT_APIC_WRITE:
13318	case VMX_EXIT_RDRAND:
13319	case VMX_EXIT_RSM:
13320	case VMX_EXIT_VMFUNC:
13321	case VMX_EXIT_ENCLS:
13322	case VMX_EXIT_RDSEED:
13323	case VMX_EXIT_XSAVES:
13324	case VMX_EXIT_XRSTORS:
13325	case VMX_EXIT_UMWAIT:
13326	case VMX_EXIT_TPAUSE:
13327	default:
13328	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13329	}
13330	#undef VMEXIT_CALL_RET
13331	}
13332	#endif /* !HMVMX_USE_FUNCTION_TABLE */
13333
13334
13335	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13336	/**
13337	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13338	*
13339	* @returns Strict VBox status code (i.e. informational status codes too).
13340	* @param pVCpu The cross context virtual CPU structure.
13341	* @param pVmxTransient The VMX-transient structure.
13342	*/
13343	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13344	{
13345	uint32_t const uExitReason = pVmxTransient->uExitReason;
13346	switch (uExitReason)
13347	{
13348	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13349	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13350	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13351	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13352	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13353
13354	/*
13355	* We shouldn't direct host physical interrupts to the nested-guest.
13356	*/
13357	case VMX_EXIT_EXT_INT:
13358	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13359
13360	/*
13361	* Instructions that cause VM-exits unconditionally or the condition is
13362	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13363	* happens, it's guaranteed to be a nested-guest VM-exit).
13364	*
13365	* - Provides VM-exit instruction length ONLY.
13366	*/
13367	case VMX_EXIT_CPUID: /* Unconditional. */
13368	case VMX_EXIT_VMCALL:
13369	case VMX_EXIT_GETSEC:
13370	case VMX_EXIT_INVD:
13371	case VMX_EXIT_XSETBV:
13372	case VMX_EXIT_VMLAUNCH:
13373	case VMX_EXIT_VMRESUME:
13374	case VMX_EXIT_VMXOFF:
13375	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13376	case VMX_EXIT_VMFUNC:
13377	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13378
13379	/*
13380	* Instructions that cause VM-exits unconditionally or the condition is
13381	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13382	* happens, it's guaranteed to be a nested-guest VM-exit).
13383	*
13384	* - Provides VM-exit instruction length.
13385	* - Provides VM-exit information.
13386	* - Optionally provides Exit qualification.
13387	*
13388	* Since Exit qualification is 0 for all VM-exits where it is not
13389	* applicable, reading and passing it to the guest should produce
13390	* defined behavior.
13391	*
13392	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
13393	*/
13394	case VMX_EXIT_INVEPT: /* Unconditional. */
13395	case VMX_EXIT_INVVPID:
13396	case VMX_EXIT_VMCLEAR:
13397	case VMX_EXIT_VMPTRLD:
13398	case VMX_EXIT_VMPTRST:
13399	case VMX_EXIT_VMXON:
13400	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13401	case VMX_EXIT_LDTR_TR_ACCESS:
13402	case VMX_EXIT_RDRAND:
13403	case VMX_EXIT_RDSEED:
13404	case VMX_EXIT_XSAVES:
13405	case VMX_EXIT_XRSTORS:
13406	case VMX_EXIT_UMWAIT:
13407	case VMX_EXIT_TPAUSE:
13408	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13409
13410	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13411	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13412	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13413	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13414	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13415	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13416	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13417	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13418	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13419	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13420	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13421	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13422	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13423	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13424	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13425	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13426	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13427	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13428	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13429
13430	case VMX_EXIT_PREEMPT_TIMER:
13431	{
13432	/** @todo NSTVMX: Preempt timer. */
13433	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
13434	}
13435
13436	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13437	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13438
13439	case VMX_EXIT_VMREAD:
13440	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13441
13442	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13443	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13444
13445	case VMX_EXIT_INIT_SIGNAL:
13446	case VMX_EXIT_SIPI:
13447	case VMX_EXIT_IO_SMI:
13448	case VMX_EXIT_SMI:
13449	case VMX_EXIT_ERR_MSR_LOAD:
13450	case VMX_EXIT_ERR_MACHINE_CHECK:
13451	case VMX_EXIT_PML_FULL:
13452	case VMX_EXIT_RSM:
13453	default:
13454	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13455	}
13456	}
13457	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13458
13459
13460	/** @name VM-exit helpers.
13461	* @{
13462	*/
13463	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13464	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13465	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13466
13467	/** Macro for VM-exits called unexpectedly. */
13468	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13469	do { \
13470	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13471	return VERR_VMX_UNEXPECTED_EXIT; \
13472	} while (0)
13473
13474	#ifdef VBOX_STRICT
13475	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13476	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13477	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13478
13479	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13480	do { \
13481	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13482	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13483	} while (0)
13484
13485	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13486	do { \
13487	AssertPtr((a_pVCpu)); \
13488	AssertPtr((a_pVmxTransient)); \
13489	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13490	Assert((a_pVmxTransient)->pVmcsInfo); \
13491	Assert(ASMIntAreEnabled()); \
13492	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13493	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13494	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13495	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13496	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13497	HMVMX_ASSERT_PREEMPT_CPUID(); \
13498	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13499	} while (0)
13500
13501	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13502	do { \
13503	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13504	Assert((a_pVmxTransient)->fIsNestedGuest); \
13505	} while (0)
13506
13507	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13508	do { \
13509	Log4Func(("\n")); \
13510	} while (0)
13511	#else
13512	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13513	do { \
13514	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13515	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13516	} while (0)
13517
13518	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13519	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13520
13521	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13522	#endif
13523
13524	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13525	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13526	* guests that attempted to execute a VMX instruction. */
13527	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13528	do \
13529	{ \
13530	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13531	if (rcStrictTmp == VINF_SUCCESS) \
13532	{ /* likely */ } \
13533	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13534	{ \
13535	Assert((a_pVCpu)->hm.s.Event.fPending); \
13536	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13537	return VINF_SUCCESS; \
13538	} \
13539	else \
13540	{ \
13541	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13542	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13543	} \
13544	} while (0)
13545
13546	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13547	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13548	do \
13549	{ \
13550	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13551	(a_pGCPtrEffAddr)); \
13552	if (rcStrictTmp == VINF_SUCCESS) \
13553	{ /* likely */ } \
13554	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13555	{ \
13556	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13557	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13558	NOREF(uXcptTmp); \
13559	return VINF_SUCCESS; \
13560	} \
13561	else \
13562	{ \
13563	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13564	return rcStrictTmp; \
13565	} \
13566	} while (0)
13567	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13568
13569
13570	/**
13571	* Advances the guest RIP by the specified number of bytes.
13572	*
13573	* @param pVCpu The cross context virtual CPU structure.
13574	* @param cbInstr Number of bytes to advance the RIP by.
13575	*
13576	* @remarks No-long-jump zone!!!
13577	*/
13578	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13579	{
13580	/* Advance the RIP. */
13581	pVCpu->cpum.GstCtx.rip += cbInstr;
13582	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13583
13584	/* Update interrupt inhibition. */
13585	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13586	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13587	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13588	}
13589
13590
13591	/**
13592	* Advances the guest RIP after reading it from the VMCS.
13593	*
13594	* @returns VBox status code, no informational status codes.
13595	* @param pVCpu The cross context virtual CPU structure.
13596	* @param pVmxTransient The VMX-transient structure.
13597	*
13598	* @remarks No-long-jump zone!!!
13599	*/
13600	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13601	{
13602	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13603	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13604	AssertRCReturn(rc, rc);
13605
13606	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13607	return VINF_SUCCESS;
13608	}
13609
13610
13611	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13612	/**
13613	* Handle a condition that occurred while delivering an event through the
13614	* nested-guest IDT.
13615	*
13616	* @returns VBox status code.
13617	* @param pVCpu The cross context virtual CPU structure.
13618	* @param pVmxTransient The VMX-transient structure.
13619	*
13620	* @remarks No-long-jump zone!!!
13621	*/
13622	static int hmR0VmxCheckExitDueToEventDeliveryNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13623	{
13624	Assert(pVmxTransient->fIsNestedGuest);
13625	Assert(!pVCpu->hm.s.Event.fPending);
13626
13627	/*
13628	* Construct a pending event from IDT vectoring information.
13629	*
13630	* This event could have originated from an event that we or the guest hypervisor injected
13631	* during nested-guest VM-entry or could arise from hardware-assisted VMX execution of the
13632	* nested-guest (for e.g. a #GP fault causing a #PF VM-exit).
13633	*
13634	* If the VM-exit is caused indirectly due to delivery of:
13635	* - #PF: the CPU would have updated CR2.
13636	* - NMI: NMI/virtual-NMI blocking is in effect.
13637	*
13638	* The main differences between this function and its non-nested version are as follows:
13639	*
13640	* - Here we record software interrupts, software exceptions and privileged software
13641	* exceptions as pending for re-injection when necessary along with gathering the
13642	* instruction length. The non-nested version would fix-up the VM-exit that occurred
13643	* during delivery of such an event and restart execution of the guest without
13644	* re-injecting the event and does not record the instruction length.
13645	*
13646	* - Here we record #PF as pending for re-injection while the non-nested version would
13647	* handle it via the page-fault VM-exit handler which isn't required when nested paging
13648	* is a requirement for hardware-assisted VMX execution of nested-guests.
13649	*
13650	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
13651	*/
13652	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13653	AssertRCReturn(rc, rc);
13654
13655	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13656	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13657	{
13658	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13659	uint8_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13660
13661	/*
13662	* Get the nasty stuff out of the way.
13663	*/
13664	{
13665	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13666	AssertRCReturn(rc, rc);
13667
13668	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13669	if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13670	{
13671	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13672	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13673	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13674
13675	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13676	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13677
13678	IEMXCPTRAISEINFO fRaiseInfo;
13679	IEMXCPTRAISE const enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags,
13680	uExitVector, &fRaiseInfo);
13681	if (enmRaise == IEMXCPTRAISE_CPU_HANG)
13682	{
13683	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13684	return VERR_EM_GUEST_CPU_HANG;
13685	}
13686	}
13687	}
13688
13689	/*
13690	* Things look legit, continue...
13691	*/
13692	uint32_t u32ErrCode;
13693	bool const fErrCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo);
13694	if (fErrCodeValid)
13695	{
13696	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13697	AssertRCReturn(rc, rc);
13698	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13699	}
13700	else
13701	u32ErrCode = 0;
13702
13703	uint32_t cbInstr;
13704	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13705	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT
13706	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT)
13707	{
13708	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13709	AssertRCReturn(rc, rc);
13710	cbInstr = pVmxTransient->cbInstr;
13711	}
13712	else
13713	cbInstr = 0;
13714
13715	RTGCUINTPTR GCPtrFaultAddress;
13716	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(uIdtVectorInfo))
13717	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
13718	else
13719	GCPtrFaultAddress = 0;
13720
13721	if (VMX_IDT_VECTORING_INFO_IS_XCPT_NMI(uIdtVectorInfo))
13722	CPUMSetGuestNmiBlocking(pVCpu, true);
13723
13724	hmR0VmxSetPendingEvent(pVCpu, uIdtVectorInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
13725	}
13726
13727	return VINF_SUCCESS;
13728	}
13729	#endif
13730
13731
13732	/**
13733	* Handle a condition that occurred while delivering an event through the guest or
13734	* nested-guest IDT.
13735	*
13736	* @returns Strict VBox status code (i.e. informational status codes too).
13737	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13738	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13739	* to continue execution of the guest which will delivery the \#DF.
13740	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13741	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13742	*
13743	* @param pVCpu The cross context virtual CPU structure.
13744	* @param pVmxTransient The VMX-transient structure.
13745	*
13746	* @remarks No-long-jump zone!!!
13747	*/
13748	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13749	{
13750	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13751	if (pVmxTransient->fIsNestedGuest)
13752	return hmR0VmxCheckExitDueToEventDeliveryNested(pVCpu, pVmxTransient);
13753	#endif
13754
13755	Assert(!pVmxTransient->fIsNestedGuest);
13756	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13757
13758	/* Read the IDT vectoring info. and VM-exit interruption info. */
13759	{
13760	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13761	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13762	AssertRCReturn(rc, rc);
13763	}
13764
13765	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13766	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13767	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13768	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13769	{
13770	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13771	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13772
13773	/*
13774	* If the event was a software interrupt (generated with INT n) or a software exception
13775	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13776	* can handle the VM-exit and continue guest execution which will re-execute the
13777	* instruction rather than re-injecting the exception, as that can cause premature
13778	* trips to ring-3 before injection and involve TRPM which currently has no way of
13779	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13780	* the problem).
13781	*/
13782	IEMXCPTRAISE enmRaise;
13783	IEMXCPTRAISEINFO fRaiseInfo;
13784	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13785	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13786	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13787	{
13788	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13789	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13790	}
13791	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13792	{
13793	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13794	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13795
13796	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13797	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13798
13799	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13800
13801	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13802	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13803	{
13804	pVmxTransient->fVectoringPF = true;
13805	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13806	}
13807	}
13808	else
13809	{
13810	/*
13811	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13812	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13813	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13814	*/
13815	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13816	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13817	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13818	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13819	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13820	}
13821
13822	/*
13823	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13824	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13825	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13826	* subsequent VM-entry would fail, see @bugref{7445}.
13827	*
13828	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13829	*/
13830	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13831	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13832	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13833	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13834	&& CPUMIsGuestNmiBlocking(pVCpu))
13835	{
13836	CPUMSetGuestNmiBlocking(pVCpu, false);
13837	}
13838
13839	switch (enmRaise)
13840	{
13841	case IEMXCPTRAISE_CURRENT_XCPT:
13842	{
13843	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", uIdtVectorInfo,
13844	uExitIntInfo));
13845	Assert(rcStrict == VINF_SUCCESS);
13846	break;
13847	}
13848
13849	case IEMXCPTRAISE_PREV_EVENT:
13850	{
13851	uint32_t u32ErrCode;
13852	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13853	{
13854	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13855	AssertRCReturn(rc, rc);
13856	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13857	}
13858	else
13859	u32ErrCode = 0;
13860
13861	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13862	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13863	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13864	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13865
13866	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13867	pVCpu->hm.s.Event.u32ErrCode));
13868	Assert(rcStrict == VINF_SUCCESS);
13869	break;
13870	}
13871
13872	case IEMXCPTRAISE_REEXEC_INSTR:
13873	Assert(rcStrict == VINF_SUCCESS);
13874	break;
13875
13876	case IEMXCPTRAISE_DOUBLE_FAULT:
13877	{
13878	/*
13879	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13880	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13881	*/
13882	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13883	{
13884	pVmxTransient->fVectoringDoublePF = true;
13885	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13886	pVCpu->cpum.GstCtx.cr2));
13887	rcStrict = VINF_SUCCESS;
13888	}
13889	else
13890	{
13891	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13892	hmR0VmxSetPendingXcptDF(pVCpu);
13893	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13894	uIdtVector, uExitVector));
13895	rcStrict = VINF_HM_DOUBLE_FAULT;
13896	}
13897	break;
13898	}
13899
13900	case IEMXCPTRAISE_TRIPLE_FAULT:
13901	{
13902	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13903	rcStrict = VINF_EM_RESET;
13904	break;
13905	}
13906
13907	case IEMXCPTRAISE_CPU_HANG:
13908	{
13909	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13910	rcStrict = VERR_EM_GUEST_CPU_HANG;
13911	break;
13912	}
13913
13914	default:
13915	{
13916	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13917	rcStrict = VERR_VMX_IPE_2;
13918	break;
13919	}
13920	}
13921	}
13922	else if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13923	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo)
13924	&& uExitVector != X86_XCPT_DF
13925	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13926	{
13927	Assert(!VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo));
13928
13929	/*
13930	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13931	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13932	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13933	*/
13934	CPUMSetGuestNmiBlocking(pVCpu, true);
13935	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13936	}
13937
13938	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13939	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13940	return rcStrict;
13941	}
13942
13943
13944	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13945	/**
13946	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13947	* guest attempting to execute a VMX instruction.
13948	*
13949	* @returns Strict VBox status code (i.e. informational status codes too).
13950	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13951	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13952	*
13953	* @param pVCpu The cross context virtual CPU structure.
13954	* @param uExitReason The VM-exit reason.
13955	*
13956	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13957	* @remarks No-long-jump zone!!!
13958	*/
13959	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13960	{
13961	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13962	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13963
13964	/*
13965	* The physical CPU would have already checked the CPU mode/code segment.
13966	* We shall just assert here for paranoia.
13967	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13968	*/
13969	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13970	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13971	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13972
13973	if (uExitReason == VMX_EXIT_VMXON)
13974	{
13975	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13976
13977	/*
13978	* We check CR4.VMXE because it is required to be always set while in VMX operation
13979	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13980	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13981	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13982	*/
13983	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13984	{
13985	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13986	hmR0VmxSetPendingXcptUD(pVCpu);
13987	return VINF_HM_PENDING_XCPT;
13988	}
13989	}
13990	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13991	{
13992	/*
13993	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13994	* (other than VMXON), we need to raise a #UD.
13995	*/
13996	Log4Func(("Not in VMX root mode -> #UD\n"));
13997	hmR0VmxSetPendingXcptUD(pVCpu);
13998	return VINF_HM_PENDING_XCPT;
13999	}
14000
14001	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
14002	return VINF_SUCCESS;
14003	}
14004
14005	/**
14006	* Decodes the memory operand of an instruction that caused a VM-exit.
14007	*
14008	* The Exit qualification field provides the displacement field for memory
14009	* operand instructions, if any.
14010	*
14011	* @returns Strict VBox status code (i.e. informational status codes too).
14012	* @retval VINF_SUCCESS if the operand was successfully decoded.
14013	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
14014	* operand.
14015	* @param pVCpu The cross context virtual CPU structure.
14016	* @param uExitInstrInfo The VM-exit instruction information field.
14017	* @param enmMemAccess The memory operand's access type (read or write).
14018	* @param GCPtrDisp The instruction displacement field, if any. For
14019	* RIP-relative addressing pass RIP + displacement here.
14020	* @param pGCPtrMem Where to store the effective destination memory address.
14021	*
14022	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
14023	* virtual-8086 mode hence skips those checks while verifying if the
14024	* segment is valid.
14025	*/
14026	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
14027	PRTGCPTR pGCPtrMem)
14028	{
14029	Assert(pGCPtrMem);
14030	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
14031	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
14032	\| CPUMCTX_EXTRN_CR0);
14033
14034	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
14035	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
14036	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
14037
14038	VMXEXITINSTRINFO ExitInstrInfo;
14039	ExitInstrInfo.u = uExitInstrInfo;
14040	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
14041	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
14042	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
14043	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
14044	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
14045	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
14046	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
14047	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
14048	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
14049
14050	/*
14051	* Validate instruction information.
14052	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
14053	*/
14054	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
14055	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
14056	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
14057	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
14058	AssertLogRelMsgReturn(fIsMemOperand,
14059	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
14060
14061	/*
14062	* Compute the complete effective address.
14063	*
14064	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
14065	* See AMD spec. 4.5.2 "Segment Registers".
14066	*/
14067	RTGCPTR GCPtrMem = GCPtrDisp;
14068	if (fBaseRegValid)
14069	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
14070	if (fIdxRegValid)
14071	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
14072
14073	RTGCPTR const GCPtrOff = GCPtrMem;
14074	if ( !fIsLongMode
14075	\|\| iSegReg >= X86_SREG_FS)
14076	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
14077	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
14078
14079	/*
14080	* Validate effective address.
14081	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
14082	*/
14083	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
14084	Assert(cbAccess > 0);
14085	if (fIsLongMode)
14086	{
14087	if (X86_IS_CANONICAL(GCPtrMem))
14088	{
14089	*pGCPtrMem = GCPtrMem;
14090	return VINF_SUCCESS;
14091	}
14092
14093	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
14094	* "Data Limit Checks in 64-bit Mode". */
14095	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
14096	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14097	return VINF_HM_PENDING_XCPT;
14098	}
14099
14100	/*
14101	* This is a watered down version of iemMemApplySegment().
14102	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
14103	* and segment CPL/DPL checks are skipped.
14104	*/
14105	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
14106	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
14107	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
14108
14109	/* Check if the segment is present and usable. */
14110	if ( pSel->Attr.n.u1Present
14111	&& !pSel->Attr.n.u1Unusable)
14112	{
14113	Assert(pSel->Attr.n.u1DescType);
14114	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
14115	{
14116	/* Check permissions for the data segment. */
14117	if ( enmMemAccess == VMXMEMACCESS_WRITE
14118	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
14119	{
14120	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14121	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
14122	return VINF_HM_PENDING_XCPT;
14123	}
14124
14125	/* Check limits if it's a normal data segment. */
14126	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
14127	{
14128	if ( GCPtrFirst32 > pSel->u32Limit
14129	\|\| GCPtrLast32 > pSel->u32Limit)
14130	{
14131	Log4Func(("Data segment limit exceeded. "
14132	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14133	GCPtrLast32, pSel->u32Limit));
14134	if (iSegReg == X86_SREG_SS)
14135	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14136	else
14137	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14138	return VINF_HM_PENDING_XCPT;
14139	}
14140	}
14141	else
14142	{
14143	/* Check limits if it's an expand-down data segment.
14144	Note! The upper boundary is defined by the B bit, not the G bit! */
14145	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
14146	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
14147	{
14148	Log4Func(("Expand-down data segment limit exceeded. "
14149	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14150	GCPtrLast32, pSel->u32Limit));
14151	if (iSegReg == X86_SREG_SS)
14152	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14153	else
14154	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14155	return VINF_HM_PENDING_XCPT;
14156	}
14157	}
14158	}
14159	else
14160	{
14161	/* Check permissions for the code segment. */
14162	if ( enmMemAccess == VMXMEMACCESS_WRITE
14163	\|\| ( enmMemAccess == VMXMEMACCESS_READ
14164	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
14165	{
14166	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
14167	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
14168	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14169	return VINF_HM_PENDING_XCPT;
14170	}
14171
14172	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
14173	if ( GCPtrFirst32 > pSel->u32Limit
14174	\|\| GCPtrLast32 > pSel->u32Limit)
14175	{
14176	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
14177	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
14178	if (iSegReg == X86_SREG_SS)
14179	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14180	else
14181	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14182	return VINF_HM_PENDING_XCPT;
14183	}
14184	}
14185	}
14186	else
14187	{
14188	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14189	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14190	return VINF_HM_PENDING_XCPT;
14191	}
14192
14193	*pGCPtrMem = GCPtrMem;
14194	return VINF_SUCCESS;
14195	}
14196	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14197
14198
14199	/**
14200	* VM-exit helper for LMSW.
14201	*/
14202	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
14203	{
14204	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14205	AssertRCReturn(rc, rc);
14206
14207	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14208	AssertMsg( rcStrict == VINF_SUCCESS
14209	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14210
14211	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14212	if (rcStrict == VINF_IEM_RAISED_XCPT)
14213	{
14214	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14215	rcStrict = VINF_SUCCESS;
14216	}
14217
14218	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14219	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14220	return rcStrict;
14221	}
14222
14223
14224	/**
14225	* VM-exit helper for CLTS.
14226	*/
14227	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
14228	{
14229	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14230	AssertRCReturn(rc, rc);
14231
14232	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
14233	AssertMsg( rcStrict == VINF_SUCCESS
14234	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14235
14236	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14237	if (rcStrict == VINF_IEM_RAISED_XCPT)
14238	{
14239	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14240	rcStrict = VINF_SUCCESS;
14241	}
14242
14243	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14244	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14245	return rcStrict;
14246	}
14247
14248
14249	/**
14250	* VM-exit helper for MOV from CRx (CRx read).
14251	*/
14252	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14253	{
14254	Assert(iCrReg < 16);
14255	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14256
14257	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14258	AssertRCReturn(rc, rc);
14259
14260	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
14261	AssertMsg( rcStrict == VINF_SUCCESS
14262	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14263
14264	if (iGReg == X86_GREG_xSP)
14265	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14266	else
14267	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14268	#ifdef VBOX_WITH_STATISTICS
14269	switch (iCrReg)
14270	{
14271	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14272	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14273	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14274	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14275	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14276	}
14277	#endif
14278	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
14279	return rcStrict;
14280	}
14281
14282
14283	/**
14284	* VM-exit helper for MOV to CRx (CRx write).
14285	*/
14286	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14287	{
14288	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14289	AssertRCReturn(rc, rc);
14290
14291	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
14292	AssertMsg( rcStrict == VINF_SUCCESS
14293	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14294	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14295
14296	switch (iCrReg)
14297	{
14298	case 0:
14299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14301	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14302	break;
14303
14304	case 2:
14305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14306	/* Nothing to do here, CR2 it's not part of the VMCS. */
14307	break;
14308
14309	case 3:
14310	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14312	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14313	break;
14314
14315	case 4:
14316	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14318	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14319	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14320	break;
14321
14322	case 8:
14323	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14324	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14325	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14326	break;
14327
14328	default:
14329	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
14330	break;
14331	}
14332
14333	if (rcStrict == VINF_IEM_RAISED_XCPT)
14334	{
14335	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14336	rcStrict = VINF_SUCCESS;
14337	}
14338	return rcStrict;
14339	}
14340
14341
14342	/**
14343	* VM-exit exception handler for \#PF (Page-fault exception).
14344	*/
14345	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14346	{
14347	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14348	PVM pVM = pVCpu->CTX_SUFF(pVM);
14349	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14350	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14351	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14352	AssertRCReturn(rc, rc);
14353
14354	if (!pVM->hm.s.fNestedPaging)
14355	{ /* likely */ }
14356	else
14357	{
14358	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14359	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
14360	#endif
14361	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14362	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14363	{
14364	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14365	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
14366	}
14367	else
14368	{
14369	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14370	hmR0VmxSetPendingXcptDF(pVCpu);
14371	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
14372	}
14373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14374	return rc;
14375	}
14376
14377	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14378	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14379	if (pVmxTransient->fVectoringPF)
14380	{
14381	Assert(pVCpu->hm.s.Event.fPending);
14382	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14383	}
14384
14385	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14386	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14387	AssertRCReturn(rc, rc);
14388
14389	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
14390	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
14391
14392	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14393	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
14394
14395	Log4Func(("#PF: rc=%Rrc\n", rc));
14396	if (rc == VINF_SUCCESS)
14397	{
14398	/*
14399	* This is typically a shadow page table sync or a MMIO instruction. But we may have
14400	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14401	*/
14402	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14403	TRPMResetTrap(pVCpu);
14404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14405	return rc;
14406	}
14407
14408	if (rc == VINF_EM_RAW_GUEST_TRAP)
14409	{
14410	if (!pVmxTransient->fVectoringDoublePF)
14411	{
14412	/* It's a guest page fault and needs to be reflected to the guest. */
14413	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14414	TRPMResetTrap(pVCpu);
14415	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14416	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14417	uGstErrorCode, pVmxTransient->uExitQual);
14418	}
14419	else
14420	{
14421	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14422	TRPMResetTrap(pVCpu);
14423	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14424	hmR0VmxSetPendingXcptDF(pVCpu);
14425	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
14426	}
14427
14428	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14429	return VINF_SUCCESS;
14430	}
14431
14432	TRPMResetTrap(pVCpu);
14433	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14434	return rc;
14435	}
14436
14437
14438	/**
14439	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14440	*/
14441	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14442	{
14443	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14444	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14445
14446	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14447	AssertRCReturn(rc, rc);
14448
14449	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14450	{
14451	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14452	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14453
14454	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14455	* provides VM-exit instruction length. If this causes problem later,
14456	* disassemble the instruction like it's done on AMD-V. */
14457	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14458	AssertRCReturn(rc2, rc2);
14459	return rc;
14460	}
14461
14462	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14463	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14464	return rc;
14465	}
14466
14467
14468	/**
14469	* VM-exit exception handler for \#BP (Breakpoint exception).
14470	*/
14471	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14472	{
14473	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14474	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14475
14476	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14477	AssertRCReturn(rc, rc);
14478
14479	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14480	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14481	if (rc == VINF_EM_RAW_GUEST_TRAP)
14482	{
14483	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14484	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14485	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14486	AssertRCReturn(rc, rc);
14487
14488	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14489	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14490	}
14491
14492	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14493	return rc;
14494	}
14495
14496
14497	/**
14498	* VM-exit exception handler for \#AC (alignment check exception).
14499	*/
14500	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14501	{
14502	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
14504
14505	/*
14506	* Re-inject it. We'll detect any nesting before getting here.
14507	*/
14508	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14509	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14510	AssertRCReturn(rc, rc);
14511	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14512
14513	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14514	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14515	return VINF_SUCCESS;
14516	}
14517
14518
14519	/**
14520	* VM-exit exception handler for \#DB (Debug exception).
14521	*/
14522	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14523	{
14524	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14526
14527	/*
14528	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
14529	*/
14530	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14531
14532	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14533	uint64_t const uDR6 = X86_DR6_INIT_VAL
14534	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14535	\| X86_DR6_BD \| X86_DR6_BS));
14536
14537	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14538	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14539	Log6Func(("rc=%Rrc\n", rc));
14540	if (rc == VINF_EM_RAW_GUEST_TRAP)
14541	{
14542	/*
14543	* The exception was for the guest. Update DR6, DR7.GD and
14544	* IA32_DEBUGCTL.LBR before forwarding it.
14545	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14546	*/
14547	VMMRZCallRing3Disable(pVCpu);
14548	HM_DISABLE_PREEMPT(pVCpu);
14549
14550	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14551	pCtx->dr[6] \|= uDR6;
14552	if (CPUMIsGuestDebugStateActive(pVCpu))
14553	ASMSetDR6(pCtx->dr[6]);
14554
14555	HM_RESTORE_PREEMPT();
14556	VMMRZCallRing3Enable(pVCpu);
14557
14558	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14559	AssertRCReturn(rc, rc);
14560
14561	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14562	pCtx->dr[7] &= ~X86_DR7_GD;
14563
14564	/* Paranoia. */
14565	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14566	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14567
14568	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14569	AssertRCReturn(rc, rc);
14570
14571	/*
14572	* Raise #DB in the guest.
14573	*
14574	* It is important to reflect exactly what the VM-exit gave us (preserving the
14575	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14576	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14577	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14578	*
14579	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14580	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14581	*/
14582	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14583	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14584	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14585	AssertRCReturn(rc, rc);
14586	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14587	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14588	return VINF_SUCCESS;
14589	}
14590
14591	/*
14592	* Not a guest trap, must be a hypervisor related debug event then.
14593	* Update DR6 in case someone is interested in it.
14594	*/
14595	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14596	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14597	CPUMSetHyperDR6(pVCpu, uDR6);
14598
14599	return rc;
14600	}
14601
14602
14603	/**
14604	* Hacks its way around the lovely mesa driver's backdoor accesses.
14605	*
14606	* @sa hmR0SvmHandleMesaDrvGp.
14607	*/
14608	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14609	{
14610	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14611	RT_NOREF(pCtx);
14612
14613	/* For now we'll just skip the instruction. */
14614	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14615	}
14616
14617
14618	/**
14619	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14620	* backdoor logging w/o checking what it is running inside.
14621	*
14622	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14623	* backdoor port and magic numbers loaded in registers.
14624	*
14625	* @returns true if it is, false if it isn't.
14626	* @sa hmR0SvmIsMesaDrvGp.
14627	*/
14628	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14629	{
14630	/* 0xed: IN eAX,dx */
14631	uint8_t abInstr[1];
14632	if (pVmxTransient->cbInstr != sizeof(abInstr))
14633	return false;
14634
14635	/* Check that it is #GP(0). */
14636	if (pVmxTransient->uExitIntErrorCode != 0)
14637	return false;
14638
14639	/* Check magic and port. */
14640	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14641	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14642	if (pCtx->rax != UINT32_C(0x564d5868))
14643	return false;
14644	if (pCtx->dx != UINT32_C(0x5658))
14645	return false;
14646
14647	/* Flat ring-3 CS. */
14648	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14649	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14650	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14651	if (pCtx->cs.Attr.n.u2Dpl != 3)
14652	return false;
14653	if (pCtx->cs.u64Base != 0)
14654	return false;
14655
14656	/* Check opcode. */
14657	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14658	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14659	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14660	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14661	if (RT_FAILURE(rc))
14662	return false;
14663	if (abInstr[0] != 0xed)
14664	return false;
14665
14666	return true;
14667	}
14668
14669	/**
14670	* VM-exit exception handler for \#GP (General-protection exception).
14671	*
14672	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14673	*/
14674	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14675	{
14676	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14678
14679	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14680	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14681	if (pVmcsInfo->RealMode.fRealOnV86Active)
14682	{ /* likely */ }
14683	else
14684	{
14685	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14686	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14687	#endif
14688	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14689	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14690	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14691	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14692	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14693	AssertRCReturn(rc, rc);
14694	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14695	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14696
14697	if ( pVmxTransient->fIsNestedGuest
14698	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14699	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14700	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14701	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14702	else
14703	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14704	return rc;
14705	}
14706
14707	Assert(CPUMIsGuestInRealModeEx(pCtx));
14708	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14709	Assert(!pVmxTransient->fIsNestedGuest);
14710
14711	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14712	AssertRCReturn(rc, rc);
14713
14714	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14715	if (rcStrict == VINF_SUCCESS)
14716	{
14717	if (!CPUMIsGuestInRealModeEx(pCtx))
14718	{
14719	/*
14720	* The guest is no longer in real-mode, check if we can continue executing the
14721	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14722	*/
14723	pVmcsInfo->RealMode.fRealOnV86Active = false;
14724	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14725	{
14726	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14727	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14728	}
14729	else
14730	{
14731	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14732	rcStrict = VINF_EM_RESCHEDULE;
14733	}
14734	}
14735	else
14736	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14737	}
14738	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14739	{
14740	rcStrict = VINF_SUCCESS;
14741	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14742	}
14743	return VBOXSTRICTRC_VAL(rcStrict);
14744	}
14745
14746
14747	/**
14748	* VM-exit exception handler wrapper for generic exceptions.
14749	*
14750	* This simply re-injects the exception back into the VM without any special
14751	* processing.
14752	*/
14753	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14754	{
14755	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14756
14757	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14758	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14759	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14760	("uVector=%#x u32XcptBitmap=%#X32\n",
14761	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14762	NOREF(pVmcsInfo);
14763	#endif
14764
14765	/*
14766	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14767	* would have been handled while checking exits due to event delivery.
14768	*/
14769	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14770	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14771	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14772	AssertRCReturn(rc, rc);
14773
14774	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14775
14776	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14777	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14778	AssertRCReturn(rc, rc);
14779	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14780	#endif
14781
14782	#ifdef VBOX_WITH_STATISTICS
14783	switch (uVector)
14784	{
14785	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14786	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14787	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14788	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14789	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14790	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14791	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14792	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14793	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14794	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14795	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14796	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14797	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14798	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14799	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14800	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14801	default:
14802	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14803	break;
14804	}
14805	#endif
14806
14807	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14808	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14809	NOREF(uVector);
14810
14811	/* Re-inject the original exception into the guest. */
14812	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14813	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14814	return VINF_SUCCESS;
14815	}
14816	/** @} */
14817
14818
14819	/** @name VM-exit handlers.
14820	* @{
14821	*/
14822	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14823	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14824	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14825
14826	/**
14827	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14828	*/
14829	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14830	{
14831	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14833	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14834	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14835	return VINF_SUCCESS;
14836	return VINF_EM_RAW_INTERRUPT;
14837	}
14838
14839
14840	/**
14841	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14842	* VM-exit.
14843	*/
14844	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14845	{
14846	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14847	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14848
14849	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14850	AssertRCReturn(rc, rc);
14851
14852	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14853	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14854
14855	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14856	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14857	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14858	NOREF(pVmcsInfo);
14859
14860	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
14861	{
14862	/*
14863	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14864	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14865	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14866	*
14867	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14868	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
14869	*/
14870	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14871	return hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14872	}
14873
14874	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14875	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14876	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
14877	{ /* likely */ }
14878	else
14879	{
14880	if (rcStrict == VINF_HM_DOUBLE_FAULT)
14881	{
14882	Assert(pVCpu->hm.s.Event.fPending);
14883	rcStrict = VINF_SUCCESS;
14884	}
14885	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14886	return rcStrict;
14887	}
14888
14889	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
14890	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
14891	switch (uIntType)
14892	{
14893	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
14894	Assert(uVector == X86_XCPT_DB);
14895	RT_FALL_THRU();
14896	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
14897	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14898	RT_FALL_THRU();
14899	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14900	{
14901	/*
14902	* If there's any exception caused as a result of event injection, the resulting
14903	* secondary/final execption will be pending, we shall continue guest execution
14904	* after injecting the event. The page-fault case is complicated and we manually
14905	* handle any currently pending event in hmR0VmxExitXcptPF.
14906	*/
14907	if (!pVCpu->hm.s.Event.fPending)
14908	{ /* likely */ }
14909	else if (uVector != X86_XCPT_PF)
14910	{
14911	rcStrict = VINF_SUCCESS;
14912	break;
14913	}
14914
14915	switch (uVector)
14916	{
14917	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
14918	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
14919	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
14920	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
14921	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
14922	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
14923	default:
14924	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient);
14925	break;
14926	}
14927	break;
14928	}
14929
14930	default:
14931	{
14932	pVCpu->hm.s.u32HMError = uExitIntInfo;
14933	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14934	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
14935	break;
14936	}
14937	}
14938	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14939	return rcStrict;
14940	}
14941
14942
14943	/**
14944	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14945	*/
14946	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14947	{
14948	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14949
14950	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14951	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14952	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14953	AssertRCReturn(rc, rc);
14954
14955	/* Evaluate and deliver pending events and resume guest execution. */
14956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14957	return VINF_SUCCESS;
14958	}
14959
14960
14961	/**
14962	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14963	*/
14964	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14965	{
14966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14967
14968	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14969	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14970	{
14971	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14972	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14973	}
14974
14975	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14976
14977	/*
14978	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14979	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14980	*/
14981	uint32_t fIntrState;
14982	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14983	AssertRCReturn(rc, rc);
14984	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14985	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14986	{
14987	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14988	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14989
14990	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14991	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14992	AssertRCReturn(rc, rc);
14993	}
14994
14995	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14996	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14997	AssertRCReturn(rc, rc);
14998
14999	/* Evaluate and deliver pending events and resume guest execution. */
15000	return VINF_SUCCESS;
15001	}
15002
15003
15004	/**
15005	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
15006	*/
15007	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15008	{
15009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15010	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15011	}
15012
15013
15014	/**
15015	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
15016	*/
15017	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15018	{
15019	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15020	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15021	}
15022
15023
15024	/**
15025	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
15026	*/
15027	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15028	{
15029	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15030
15031	/*
15032	* Get the state we need and update the exit history entry.
15033	*/
15034	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15035	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15036	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15037	AssertRCReturn(rc, rc);
15038
15039	VBOXSTRICTRC rcStrict;
15040	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15041	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
15042	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15043	if (!pExitRec)
15044	{
15045	/*
15046	* Regular CPUID instruction execution.
15047	*/
15048	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
15049	if (rcStrict == VINF_SUCCESS)
15050	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15051	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15052	{
15053	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15054	rcStrict = VINF_SUCCESS;
15055	}
15056	}
15057	else
15058	{
15059	/*
15060	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15061	*/
15062	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15063	AssertRCReturn(rc2, rc2);
15064
15065	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
15066	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
15067
15068	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15069	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15070
15071	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15072	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15073	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15074	}
15075	return rcStrict;
15076	}
15077
15078
15079	/**
15080	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
15081	*/
15082	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15083	{
15084	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15085
15086	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15087	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
15088	AssertRCReturn(rc, rc);
15089
15090	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
15091	return VINF_EM_RAW_EMULATE_INSTR;
15092
15093	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
15094	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15095	}
15096
15097
15098	/**
15099	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
15100	*/
15101	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15102	{
15103	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15104
15105	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15106	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15107	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15108	AssertRCReturn(rc, rc);
15109
15110	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
15111	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15112	{
15113	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15114	we must reset offsetting on VM-entry. See @bugref{6634}. */
15115	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15116	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15118	}
15119	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15120	{
15121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15122	rcStrict = VINF_SUCCESS;
15123	}
15124	return rcStrict;
15125	}
15126
15127
15128	/**
15129	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
15130	*/
15131	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15132	{
15133	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15134
15135	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15136	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
15137	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15138	AssertRCReturn(rc, rc);
15139
15140	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
15141	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15142	{
15143	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15144	we must reset offsetting on VM-reentry. See @bugref{6634}. */
15145	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15146	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15147	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15148	}
15149	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15150	{
15151	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15152	rcStrict = VINF_SUCCESS;
15153	}
15154	return rcStrict;
15155	}
15156
15157
15158	/**
15159	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15160	*/
15161	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15162	{
15163	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15164
15165	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15166	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
15167	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
15168	AssertRCReturn(rc, rc);
15169
15170	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15171	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15172	if (RT_LIKELY(rc == VINF_SUCCESS))
15173	{
15174	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15175	Assert(pVmxTransient->cbInstr == 2);
15176	}
15177	else
15178	{
15179	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
15180	rc = VERR_EM_INTERPRETER;
15181	}
15182	return rc;
15183	}
15184
15185
15186	/**
15187	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
15188	*/
15189	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15190	{
15191	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15192
15193	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
15194	if (EMAreHypercallInstructionsEnabled(pVCpu))
15195	{
15196	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15197	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
15198	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
15199	AssertRCReturn(rc, rc);
15200
15201	/* Perform the hypercall. */
15202	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
15203	if (rcStrict == VINF_SUCCESS)
15204	{
15205	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15206	AssertRCReturn(rc, rc);
15207	}
15208	else
15209	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
15210	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
15211	\|\| RT_FAILURE(rcStrict));
15212
15213	/* If the hypercall changes anything other than guest's general-purpose registers,
15214	we would need to reload the guest changed bits here before VM-entry. */
15215	}
15216	else
15217	Log4Func(("Hypercalls not enabled\n"));
15218
15219	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
15220	if (RT_FAILURE(rcStrict))
15221	{
15222	hmR0VmxSetPendingXcptUD(pVCpu);
15223	rcStrict = VINF_SUCCESS;
15224	}
15225
15226	return rcStrict;
15227	}
15228
15229
15230	/**
15231	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15232	*/
15233	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15234	{
15235	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15236	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
15237
15238	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15239	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15240	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15241	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15242	AssertRCReturn(rc, rc);
15243
15244	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
15245
15246	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
15247	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15248	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15249	{
15250	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15251	rcStrict = VINF_SUCCESS;
15252	}
15253	else
15254	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
15255	VBOXSTRICTRC_VAL(rcStrict)));
15256	return rcStrict;
15257	}
15258
15259
15260	/**
15261	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
15262	*/
15263	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15264	{
15265	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15266
15267	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15268	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
15269	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15270	AssertRCReturn(rc, rc);
15271
15272	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
15273	if (rcStrict == VINF_SUCCESS)
15274	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15275	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15276	{
15277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15278	rcStrict = VINF_SUCCESS;
15279	}
15280
15281	return rcStrict;
15282	}
15283
15284
15285	/**
15286	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
15287	*/
15288	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15289	{
15290	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15291
15292	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15293	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15294	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15295	AssertRCReturn(rc, rc);
15296
15297	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
15298	if (RT_SUCCESS(rcStrict))
15299	{
15300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15301	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
15302	rcStrict = VINF_SUCCESS;
15303	}
15304
15305	return rcStrict;
15306	}
15307
15308
15309	/**
15310	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
15311	* VM-exit.
15312	*/
15313	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15314	{
15315	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15316	return VINF_EM_RESET;
15317	}
15318
15319
15320	/**
15321	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15322	*/
15323	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15324	{
15325	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15326
15327	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15328	AssertRCReturn(rc, rc);
15329
15330	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
15331	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
15332	rc = VINF_SUCCESS;
15333	else
15334	rc = VINF_EM_HALT;
15335
15336	if (rc != VINF_SUCCESS)
15337	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
15338	return rc;
15339	}
15340
15341
15342	/**
15343	* VM-exit handler for instructions that result in a \#UD exception delivered to
15344	* the guest.
15345	*/
15346	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15347	{
15348	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15349	hmR0VmxSetPendingXcptUD(pVCpu);
15350	return VINF_SUCCESS;
15351	}
15352
15353
15354	/**
15355	* VM-exit handler for expiry of the VMX-preemption timer.
15356	*/
15357	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15358	{
15359	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15360
15361	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
15362	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15363
15364	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
15365	PVM pVM = pVCpu->CTX_SUFF(pVM);
15366	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
15367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
15368	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
15369	}
15370
15371
15372	/**
15373	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
15374	*/
15375	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15376	{
15377	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15378
15379	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15380	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15381	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
15382	AssertRCReturn(rc, rc);
15383
15384	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
15385	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15386	: HM_CHANGED_RAISED_XCPT_MASK);
15387
15388	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15389	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
15390
15391	return rcStrict;
15392	}
15393
15394
15395	/**
15396	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
15397	*/
15398	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15399	{
15400	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15401	/** @todo Use VM-exit instruction information. */
15402	return VERR_EM_INTERPRETER;
15403	}
15404
15405
15406	/**
15407	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
15408	* VM-exit.
15409	*/
15410	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15411	{
15412	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15413	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15414	AssertRCReturn(rc, rc);
15415
15416	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
15417	if (RT_FAILURE(rc))
15418	return rc;
15419
15420	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
15421	NOREF(uInvalidReason);
15422
15423	#ifdef VBOX_STRICT
15424	uint32_t fIntrState;
15425	RTHCUINTREG uHCReg;
15426	uint64_t u64Val;
15427	uint32_t u32Val;
15428	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
15429	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
15430	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
15431	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
15432	AssertRCReturn(rc, rc);
15433
15434	Log4(("uInvalidReason %u\n", uInvalidReason));
15435	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
15436	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
15437	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
15438	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
15439
15440	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
15441	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
15442	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
15443	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
15444	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
15445	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15446	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
15447	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
15448	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
15449	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15450	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
15451	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
15452
15453	hmR0DumpRegs(pVCpu);
15454	#endif
15455
15456	return VERR_VMX_INVALID_GUEST_STATE;
15457	}
15458
15459	/**
15460	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
15461	*/
15462	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15463	{
15464	/*
15465	* Cummulative notes of all recognized but unexpected VM-exits.
15466	*
15467	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
15468	* nested-paging is used.
15469	*
15470	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
15471	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
15472	* this function (and thereby stop VM execution) for handling such instructions.
15473	*
15474	*
15475	* VMX_EXIT_INIT_SIGNAL:
15476	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15477	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15478	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15479	*
15480	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15481	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15482	* See Intel spec. "23.8 Restrictions on VMX operation".
15483	*
15484	* VMX_EXIT_SIPI:
15485	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15486	* activity state is used. We don't make use of it as our guests don't have direct
15487	* access to the host local APIC.
15488	*
15489	* See Intel spec. 25.3 "Other Causes of VM-exits".
15490	*
15491	* VMX_EXIT_IO_SMI:
15492	* VMX_EXIT_SMI:
15493	* This can only happen if we support dual-monitor treatment of SMI, which can be
15494	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15495	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15496	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15497	*
15498	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15499	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15500	*
15501	* VMX_EXIT_ERR_MSR_LOAD:
15502	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15503	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15504	* execution.
15505	*
15506	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15507	*
15508	* VMX_EXIT_ERR_MACHINE_CHECK:
15509	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15510	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15511	* #MC exception abort class exception is raised. We thus cannot assume a
15512	* reasonable chance of continuing any sort of execution and we bail.
15513	*
15514	* See Intel spec. 15.1 "Machine-check Architecture".
15515	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15516	*
15517	* VMX_EXIT_PML_FULL:
15518	* VMX_EXIT_VIRTUALIZED_EOI:
15519	* VMX_EXIT_APIC_WRITE:
15520	* We do not currently support any of these features and thus they are all unexpected
15521	* VM-exits.
15522	*
15523	* VMX_EXIT_GDTR_IDTR_ACCESS:
15524	* VMX_EXIT_LDTR_TR_ACCESS:
15525	* VMX_EXIT_RDRAND:
15526	* VMX_EXIT_RSM:
15527	* VMX_EXIT_VMFUNC:
15528	* VMX_EXIT_ENCLS:
15529	* VMX_EXIT_RDSEED:
15530	* VMX_EXIT_XSAVES:
15531	* VMX_EXIT_XRSTORS:
15532	* VMX_EXIT_UMWAIT:
15533	* VMX_EXIT_TPAUSE:
15534	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15535	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15536	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15537	*
15538	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15539	*/
15540	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15541	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15542	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15543	}
15544
15545
15546	/**
15547	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15548	*/
15549	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15550	{
15551	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15552
15553	/** @todo Optimize this: We currently drag in in the whole MSR state
15554	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15555	* MSRs required. That would require changes to IEM and possibly CPUM too.
15556	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15557	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15558	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15559	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15560	switch (idMsr)
15561	{
15562	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15563	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15564	}
15565
15566	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15567	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15568	AssertRCReturn(rc, rc);
15569
15570	Log4Func(("ecx=%#RX32\n", idMsr));
15571
15572	#ifdef VBOX_STRICT
15573	Assert(!pVmxTransient->fIsNestedGuest);
15574	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15575	{
15576	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15577	&& idMsr != MSR_K6_EFER)
15578	{
15579	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15580	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15581	}
15582	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15583	{
15584	Assert(pVmcsInfo->pvMsrBitmap);
15585	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15586	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15587	{
15588	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15589	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15590	}
15591	}
15592	}
15593	#endif
15594
15595	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15597	if (rcStrict == VINF_SUCCESS)
15598	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15599	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15600	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15601	{
15602	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15603	rcStrict = VINF_SUCCESS;
15604	}
15605	else
15606	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15607
15608	return rcStrict;
15609	}
15610
15611
15612	/**
15613	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15614	*/
15615	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15616	{
15617	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15618
15619	/** @todo Optimize this: We currently drag in in the whole MSR state
15620	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15621	* MSRs required. That would require changes to IEM and possibly CPUM too.
15622	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15623	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15624	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15625
15626	/*
15627	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15628	* Although we don't need to fetch the base as it will be overwritten shortly, while
15629	* loading guest-state we would also load the entire segment register including limit
15630	* and attributes and thus we need to load them here.
15631	*/
15632	switch (idMsr)
15633	{
15634	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15635	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15636	}
15637
15638	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15639	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15640	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15641	AssertRCReturn(rc, rc);
15642
15643	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15644
15645	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15647
15648	if (rcStrict == VINF_SUCCESS)
15649	{
15650	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15651
15652	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15653	if ( idMsr == MSR_IA32_APICBASE
15654	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15655	&& idMsr <= MSR_IA32_X2APIC_END))
15656	{
15657	/*
15658	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15659	* When full APIC register virtualization is implemented we'll have to make
15660	* sure APIC state is saved from the VMCS before IEM changes it.
15661	*/
15662	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15663	}
15664	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15665	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15666	else if (idMsr == MSR_K6_EFER)
15667	{
15668	/*
15669	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15670	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15671	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15672	*/
15673	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15674	}
15675
15676	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15677	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15678	{
15679	switch (idMsr)
15680	{
15681	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15682	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15683	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15684	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15685	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15686	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15687	default:
15688	{
15689	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15690	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15691	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15692	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15693	break;
15694	}
15695	}
15696	}
15697	#ifdef VBOX_STRICT
15698	else
15699	{
15700	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15701	switch (idMsr)
15702	{
15703	case MSR_IA32_SYSENTER_CS:
15704	case MSR_IA32_SYSENTER_EIP:
15705	case MSR_IA32_SYSENTER_ESP:
15706	case MSR_K8_FS_BASE:
15707	case MSR_K8_GS_BASE:
15708	{
15709	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15710	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15711	}
15712
15713	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15714	default:
15715	{
15716	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15717	{
15718	/* EFER MSR writes are always intercepted. */
15719	if (idMsr != MSR_K6_EFER)
15720	{
15721	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15722	idMsr));
15723	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15724	}
15725	}
15726
15727	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15728	{
15729	Assert(pVmcsInfo->pvMsrBitmap);
15730	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15731	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15732	{
15733	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15734	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15735	}
15736	}
15737	break;
15738	}
15739	}
15740	}
15741	#endif /* VBOX_STRICT */
15742	}
15743	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15744	{
15745	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15746	rcStrict = VINF_SUCCESS;
15747	}
15748	else
15749	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15750
15751	return rcStrict;
15752	}
15753
15754
15755	/**
15756	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15757	*/
15758	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15759	{
15760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15761
15762	/** @todo The guest has likely hit a contended spinlock. We might want to
15763	* poke a schedule different guest VCPU. */
15764	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15765	if (RT_SUCCESS(rc))
15766	return VINF_EM_RAW_INTERRUPT;
15767
15768	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15769	return rc;
15770	}
15771
15772
15773	/**
15774	* VM-exit handler for when the TPR value is lowered below the specified
15775	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15776	*/
15777	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15778	{
15779	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15780	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15781
15782	/*
15783	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15784	* We'll re-evaluate pending interrupts and inject them before the next VM
15785	* entry so we can just continue execution here.
15786	*/
15787	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15788	return VINF_SUCCESS;
15789	}
15790
15791
15792	/**
15793	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15794	* VM-exit.
15795	*
15796	* @retval VINF_SUCCESS when guest execution can continue.
15797	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15798	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15799	* incompatible guest state for VMX execution (real-on-v86 case).
15800	*/
15801	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15802	{
15803	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15804	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15805
15806	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15807	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15808	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15809	AssertRCReturn(rc, rc);
15810
15811	VBOXSTRICTRC rcStrict;
15812	PVM pVM = pVCpu->CTX_SUFF(pVM);
15813	uint64_t const uExitQual = pVmxTransient->uExitQual;
15814	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15815	switch (uAccessType)
15816	{
15817	/*
15818	* MOV to CRx.
15819	*/
15820	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15821	{
15822	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15823	AssertRCReturn(rc, rc);
15824
15825	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15826	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15827	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15828	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15829
15830	/*
15831	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15832	* - When nested paging isn't used.
15833	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15834	* - We are executing in the VM debug loop.
15835	*/
15836	Assert( iCrReg != 3
15837	\|\| !pVM->hm.s.fNestedPaging
15838	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15839	\|\| pVCpu->hm.s.fUsingDebugLoop);
15840
15841	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15842	Assert( iCrReg != 8
15843	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15844
15845	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15846	AssertMsg( rcStrict == VINF_SUCCESS
15847	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15848
15849	/*
15850	* This is a kludge for handling switches back to real mode when we try to use
15851	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15852	* deal with special selector values, so we have to return to ring-3 and run
15853	* there till the selector values are V86 mode compatible.
15854	*
15855	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15856	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15857	* this function.
15858	*/
15859	if ( iCrReg == 0
15860	&& rcStrict == VINF_SUCCESS
15861	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15862	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15863	&& (uOldCr0 & X86_CR0_PE)
15864	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15865	{
15866	/** @todo Check selectors rather than returning all the time. */
15867	Assert(!pVmxTransient->fIsNestedGuest);
15868	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15869	rcStrict = VINF_EM_RESCHEDULE_REM;
15870	}
15871	break;
15872	}
15873
15874	/*
15875	* MOV from CRx.
15876	*/
15877	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15878	{
15879	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15880	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15881
15882	/*
15883	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15884	* - When nested paging isn't used.
15885	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15886	* - We are executing in the VM debug loop.
15887	*/
15888	Assert( iCrReg != 3
15889	\|\| !pVM->hm.s.fNestedPaging
15890	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15891	\|\| pVCpu->hm.s.fUsingDebugLoop);
15892
15893	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15894	Assert( iCrReg != 8
15895	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15896
15897	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15898	break;
15899	}
15900
15901	/*
15902	* CLTS (Clear Task-Switch Flag in CR0).
15903	*/
15904	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15905	{
15906	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15907	break;
15908	}
15909
15910	/*
15911	* LMSW (Load Machine-Status Word into CR0).
15912	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15913	*/
15914	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15915	{
15916	RTGCPTR GCPtrEffDst;
15917	uint8_t const cbInstr = pVmxTransient->cbInstr;
15918	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15919	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15920	if (fMemOperand)
15921	{
15922	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15923	AssertRCReturn(rc, rc);
15924	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15925	}
15926	else
15927	GCPtrEffDst = NIL_RTGCPTR;
15928	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15929	break;
15930	}
15931
15932	default:
15933	{
15934	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15935	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15936	}
15937	}
15938
15939	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15940	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15941	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15942
15943	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15944	NOREF(pVM);
15945	return rcStrict;
15946	}
15947
15948
15949	/**
15950	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15951	* VM-exit.
15952	*/
15953	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15954	{
15955	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15956	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15957
15958	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15959	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15960	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15961	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15962	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15963	\| CPUMCTX_EXTRN_EFER);
15964	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15965	AssertRCReturn(rc, rc);
15966
15967	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15968	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15969	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15970	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15971	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15972	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15973	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15974	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15975
15976	/*
15977	* Update exit history to see if this exit can be optimized.
15978	*/
15979	VBOXSTRICTRC rcStrict;
15980	PCEMEXITREC pExitRec = NULL;
15981	if ( !fGstStepping
15982	&& !fDbgStepping)
15983	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15984	!fIOString
15985	? !fIOWrite
15986	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15987	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15988	: !fIOWrite
15989	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15990	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15991	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15992	if (!pExitRec)
15993	{
15994	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15995	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15996
15997	uint32_t const cbValue = s_aIOSizes[uIOSize];
15998	uint32_t const cbInstr = pVmxTransient->cbInstr;
15999	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
16000	PVM pVM = pVCpu->CTX_SUFF(pVM);
16001	if (fIOString)
16002	{
16003	/*
16004	* INS/OUTS - I/O String instruction.
16005	*
16006	* Use instruction-information if available, otherwise fall back on
16007	* interpreting the instruction.
16008	*/
16009	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
16010	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
16011	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
16012	if (fInsOutsInfo)
16013	{
16014	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16015	AssertRCReturn(rc2, rc2);
16016	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
16017	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
16018	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
16019	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
16020	if (fIOWrite)
16021	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
16022	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
16023	else
16024	{
16025	/*
16026	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
16027	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
16028	* See Intel Instruction spec. for "INS".
16029	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
16030	*/
16031	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
16032	}
16033	}
16034	else
16035	rcStrict = IEMExecOne(pVCpu);
16036
16037	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
16038	fUpdateRipAlready = true;
16039	}
16040	else
16041	{
16042	/*
16043	* IN/OUT - I/O instruction.
16044	*/
16045	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
16046	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
16047	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
16048	if (fIOWrite)
16049	{
16050	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
16051	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
16052	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
16053	&& !pCtx->eflags.Bits.u1TF)
16054	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
16055	}
16056	else
16057	{
16058	uint32_t u32Result = 0;
16059	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
16060	if (IOM_SUCCESS(rcStrict))
16061	{
16062	/* Save result of I/O IN instr. in AL/AX/EAX. */
16063	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
16064	}
16065	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
16066	&& !pCtx->eflags.Bits.u1TF)
16067	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
16068	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
16069	}
16070	}
16071
16072	if (IOM_SUCCESS(rcStrict))
16073	{
16074	if (!fUpdateRipAlready)
16075	{
16076	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
16077	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
16078	}
16079
16080	/*
16081	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
16082	* while booting Fedora 17 64-bit guest.
16083	*
16084	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
16085	*/
16086	if (fIOString)
16087	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
16088
16089	/*
16090	* If any I/O breakpoints are armed, we need to check if one triggered
16091	* and take appropriate action.
16092	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
16093	*/
16094	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
16095	AssertRCReturn(rc, rc);
16096
16097	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
16098	* execution engines about whether hyper BPs and such are pending. */
16099	uint32_t const uDr7 = pCtx->dr[7];
16100	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
16101	&& X86_DR7_ANY_RW_IO(uDr7)
16102	&& (pCtx->cr4 & X86_CR4_DE))
16103	\|\| DBGFBpIsHwIoArmed(pVM)))
16104	{
16105	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
16106
16107	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
16108	VMMRZCallRing3Disable(pVCpu);
16109	HM_DISABLE_PREEMPT(pVCpu);
16110
16111	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
16112
16113	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
16114	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
16115	{
16116	/* Raise #DB. */
16117	if (fIsGuestDbgActive)
16118	ASMSetDR6(pCtx->dr[6]);
16119	if (pCtx->dr[7] != uDr7)
16120	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
16121
16122	hmR0VmxSetPendingXcptDB(pVCpu);
16123	}
16124	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
16125	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
16126	else if ( rcStrict2 != VINF_SUCCESS
16127	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
16128	rcStrict = rcStrict2;
16129	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
16130
16131	HM_RESTORE_PREEMPT();
16132	VMMRZCallRing3Enable(pVCpu);
16133	}
16134	}
16135
16136	#ifdef VBOX_STRICT
16137	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
16138	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
16139	Assert(!fIOWrite);
16140	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
16141	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
16142	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
16143	Assert(fIOWrite);
16144	else
16145	{
16146	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
16147	* statuses, that the VMM device and some others may return. See
16148	* IOM_SUCCESS() for guidance. */
16149	AssertMsg( RT_FAILURE(rcStrict)
16150	\|\| rcStrict == VINF_SUCCESS
16151	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
16152	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
16153	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
16154	\|\| rcStrict == VINF_EM_RAW_TO_R3
16155	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16156	# endif
16157	}
16158	#endif
16159	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
16160	}
16161	else
16162	{
16163	/*
16164	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16165	*/
16166	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16167	AssertRCReturn(rc2, rc2);
16168	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
16169	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
16170	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
16171	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16172	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
16173	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
16174
16175	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16176	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16177
16178	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16179	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16180	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16181	}
16182	return rcStrict;
16183	}
16184
16185
16186	/**
16187	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
16188	* VM-exit.
16189	*/
16190	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16191	{
16192	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16193
16194	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
16195	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16196	AssertRCReturn(rc, rc);
16197	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
16198	{
16199	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16200	AssertRCReturn(rc, rc);
16201	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16202	{
16203	uint32_t uErrCode;
16204	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
16205	{
16206	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16207	AssertRCReturn(rc, rc);
16208	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
16209	}
16210	else
16211	uErrCode = 0;
16212
16213	RTGCUINTPTR GCPtrFaultAddress;
16214	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
16215	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
16216	else
16217	GCPtrFaultAddress = 0;
16218
16219	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16220	AssertRCReturn(rc, rc);
16221
16222	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
16223	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
16224
16225	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
16226	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
16227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16228	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16229	}
16230	}
16231
16232	/* Fall back to the interpreter to emulate the task-switch. */
16233	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16234	return VERR_EM_INTERPRETER;
16235	}
16236
16237
16238	/**
16239	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
16240	*/
16241	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16242	{
16243	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16244
16245	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16246	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
16247	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16248	AssertRCReturn(rc, rc);
16249	return VINF_EM_DBG_STEPPED;
16250	}
16251
16252
16253	/**
16254	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
16255	*/
16256	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16257	{
16258	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
16260
16261	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16262	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16263	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16264	{
16265	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
16266	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16267	{
16268	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16269	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16270	}
16271	}
16272	else
16273	{
16274	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16275	{
16276	Assert(pVCpu->hm.s.Event.fPending);
16277	rcStrict1 = VINF_SUCCESS;
16278	}
16279	return rcStrict1;
16280	}
16281
16282	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
16283	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16284	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16285	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16286	AssertRCReturn(rc, rc);
16287
16288	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
16289	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
16290	VBOXSTRICTRC rcStrict2;
16291	switch (uAccessType)
16292	{
16293	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
16294	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
16295	{
16296	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
16297	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
16298	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
16299
16300	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
16301	GCPhys &= PAGE_BASE_GC_MASK;
16302	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
16303	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
16304	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
16305
16306	PVM pVM = pVCpu->CTX_SUFF(pVM);
16307	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16308	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
16309	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
16310	CPUMCTX2CORE(pCtx), GCPhys);
16311	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16312	if ( rcStrict2 == VINF_SUCCESS
16313	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16314	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16315	{
16316	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16317	\| HM_CHANGED_GUEST_APIC_TPR);
16318	rcStrict2 = VINF_SUCCESS;
16319	}
16320	break;
16321	}
16322
16323	default:
16324	{
16325	Log4Func(("uAccessType=%#x\n", uAccessType));
16326	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
16327	break;
16328	}
16329	}
16330
16331	if (rcStrict2 != VINF_SUCCESS)
16332	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
16333	return rcStrict2;
16334	}
16335
16336
16337	/**
16338	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
16339	* VM-exit.
16340	*/
16341	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16342	{
16343	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16344
16345	/* We should -not- get this VM-exit if the guest's debug registers were active. */
16346	if (pVmxTransient->fWasGuestDebugStateActive)
16347	{
16348	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
16349	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
16350	}
16351
16352	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16353	if ( !pVCpu->hm.s.fSingleInstruction
16354	&& !pVmxTransient->fWasHyperDebugStateActive)
16355	{
16356	Assert(!DBGFIsStepping(pVCpu));
16357	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
16358
16359	/* Don't intercept MOV DRx any more. */
16360	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
16361	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16362	AssertRCReturn(rc, rc);
16363
16364	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
16365	VMMRZCallRing3Disable(pVCpu);
16366	HM_DISABLE_PREEMPT(pVCpu);
16367
16368	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
16369	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
16370	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
16371
16372	HM_RESTORE_PREEMPT();
16373	VMMRZCallRing3Enable(pVCpu);
16374
16375	#ifdef VBOX_WITH_STATISTICS
16376	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16377	AssertRCReturn(rc, rc);
16378	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16380	else
16381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16382	#endif
16383	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
16384	return VINF_SUCCESS;
16385	}
16386
16387	/*
16388	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
16389	* The EFER MSR is always up-to-date.
16390	* Update the segment registers and DR7 from the CPU.
16391	*/
16392	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16393	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16394	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
16395	AssertRCReturn(rc, rc);
16396	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
16397
16398	PVM pVM = pVCpu->CTX_SUFF(pVM);
16399	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16400	{
16401	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16402	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
16403	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
16404	if (RT_SUCCESS(rc))
16405	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
16406	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16407	}
16408	else
16409	{
16410	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16411	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
16412	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
16413	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16414	}
16415
16416	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
16417	if (RT_SUCCESS(rc))
16418	{
16419	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
16420	AssertRCReturn(rc2, rc2);
16421	return VINF_SUCCESS;
16422	}
16423	return rc;
16424	}
16425
16426
16427	/**
16428	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
16429	* Conditional VM-exit.
16430	*/
16431	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16432	{
16433	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16434	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16435
16436	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16437	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16438	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16439	{
16440	/*
16441	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
16442	* instruction emulation to inject the original event. Otherwise, injecting the original event
16443	* using hardware-assisted VMX would would trigger the same EPT misconfig VM-exit again.
16444	*/
16445	if (!pVCpu->hm.s.Event.fPending)
16446	{ /* likely */ }
16447	else
16448	{
16449	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16450	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16451	/** @todo NSTVMX: Think about how this should be handled. */
16452	if (pVmxTransient->fIsNestedGuest)
16453	return VERR_VMX_IPE_3;
16454	#endif
16455	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16456	}
16457	}
16458	else
16459	{
16460	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16461	rcStrict = VINF_SUCCESS;
16462	return rcStrict;
16463	}
16464
16465	/*
16466	* Get sufficent state and update the exit history entry.
16467	*/
16468	RTGCPHYS GCPhys;
16469	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16470	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16471	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16472	AssertRCReturn(rc, rc);
16473
16474	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16475	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16476	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16477	if (!pExitRec)
16478	{
16479	/*
16480	* If we succeed, resume guest execution.
16481	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16482	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16483	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16484	* weird case. See @bugref{6043}.
16485	*/
16486	PVM pVM = pVCpu->CTX_SUFF(pVM);
16487	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16488	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16489	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16490	if ( rcStrict == VINF_SUCCESS
16491	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16492	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16493	{
16494	/* Successfully handled MMIO operation. */
16495	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16496	\| HM_CHANGED_GUEST_APIC_TPR);
16497	rcStrict = VINF_SUCCESS;
16498	}
16499	}
16500	else
16501	{
16502	/*
16503	* Frequent exit or something needing probing. Call EMHistoryExec.
16504	*/
16505	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16506	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16507
16508	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16509	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16510
16511	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16512	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16513	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16514	}
16515	return rcStrict;
16516	}
16517
16518
16519	/**
16520	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16521	* VM-exit.
16522	*/
16523	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16524	{
16525	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16526	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16527
16528	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16529	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16530	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16531	{
16532	/*
16533	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16534	* we shall resolve the nested #PF and re-inject the original event.
16535	*/
16536	if (pVCpu->hm.s.Event.fPending)
16537	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16538	}
16539	else
16540	{
16541	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16542	{
16543	Assert(pVCpu->hm.s.Event.fPending);
16544	rcStrict = VINF_SUCCESS;
16545	}
16546	return rcStrict;
16547	}
16548
16549	RTGCPHYS GCPhys;
16550	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16551	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16552	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16553	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16554	AssertRCReturn(rc, rc);
16555
16556	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
16557	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
16558
16559	RTGCUINT uErrorCode = 0;
16560	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16561	uErrorCode \|= X86_TRAP_PF_ID;
16562	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16563	uErrorCode \|= X86_TRAP_PF_RW;
16564	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16565	uErrorCode \|= X86_TRAP_PF_P;
16566
16567	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16568
16569	/* Handle the pagefault trap for the nested shadow table. */
16570	PVM pVM = pVCpu->CTX_SUFF(pVM);
16571	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16572
16573	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
16574	pCtx->cs.Sel, pCtx->rip));
16575
16576	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16577	TRPMResetTrap(pVCpu);
16578
16579	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16580	if ( rcStrict == VINF_SUCCESS
16581	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16582	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16583	{
16584	/* Successfully synced our nested page tables. */
16585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16586	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16587	return VINF_SUCCESS;
16588	}
16589
16590	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16591	return rcStrict;
16592	}
16593
16594
16595	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16596	/**
16597	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16598	*/
16599	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16600	{
16601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16602
16603	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16604	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16605	\| CPUMCTX_EXTRN_HWVIRT
16606	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16607	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16608	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16609	AssertRCReturn(rc, rc);
16610
16611	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16612
16613	VMXVEXITINFO ExitInfo;
16614	RT_ZERO(ExitInfo);
16615	ExitInfo.uReason = pVmxTransient->uExitReason;
16616	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16617	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16618	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16619	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16620
16621	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16622	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16623	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16624	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16625	{
16626	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16627	rcStrict = VINF_SUCCESS;
16628	}
16629	return rcStrict;
16630	}
16631
16632
16633	/**
16634	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16635	*/
16636	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16637	{
16638	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16639
16640	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16641	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16642	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16643	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16644	AssertRCReturn(rc, rc);
16645
16646	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16647
16648	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16649	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16650	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16651	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16652	{
16653	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16654	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16655	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16656	}
16657	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16658	return rcStrict;
16659	}
16660
16661
16662	/**
16663	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16664	*/
16665	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16666	{
16667	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16668
16669	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16670	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16671	\| CPUMCTX_EXTRN_HWVIRT
16672	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16673	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16674	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16675	AssertRCReturn(rc, rc);
16676
16677	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16678
16679	VMXVEXITINFO ExitInfo;
16680	RT_ZERO(ExitInfo);
16681	ExitInfo.uReason = pVmxTransient->uExitReason;
16682	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16683	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16684	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16685	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16686
16687	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16688	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16689	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16690	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16691	{
16692	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16693	rcStrict = VINF_SUCCESS;
16694	}
16695	return rcStrict;
16696	}
16697
16698
16699	/**
16700	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16701	*/
16702	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16703	{
16704	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16705
16706	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16707	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16708	\| CPUMCTX_EXTRN_HWVIRT
16709	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16710	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16711	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16712	AssertRCReturn(rc, rc);
16713
16714	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16715
16716	VMXVEXITINFO ExitInfo;
16717	RT_ZERO(ExitInfo);
16718	ExitInfo.uReason = pVmxTransient->uExitReason;
16719	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16720	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16721	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16722	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16723
16724	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16725	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16726	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16727	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16728	{
16729	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16730	rcStrict = VINF_SUCCESS;
16731	}
16732	return rcStrict;
16733	}
16734
16735
16736	/**
16737	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16738	*/
16739	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16740	{
16741	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16742
16743	/*
16744	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16745	* thus might not need to import the shadow VMCS state, it's safer just in case
16746	* code elsewhere dares look at unsynced VMCS fields.
16747	*/
16748	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16749	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16750	\| CPUMCTX_EXTRN_HWVIRT
16751	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16752	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16753	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16754	AssertRCReturn(rc, rc);
16755
16756	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16757
16758	VMXVEXITINFO ExitInfo;
16759	RT_ZERO(ExitInfo);
16760	ExitInfo.uReason = pVmxTransient->uExitReason;
16761	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16762	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16763	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16764	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16765	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16766
16767	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16768	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16769	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16770	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16771	{
16772	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16773	rcStrict = VINF_SUCCESS;
16774	}
16775	return rcStrict;
16776	}
16777
16778
16779	/**
16780	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16781	*/
16782	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16783	{
16784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16785
16786	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16787	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16788	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16789	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16790	AssertRCReturn(rc, rc);
16791
16792	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16793
16794	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16795	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16796	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16797	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16798	{
16799	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16800	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16801	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16802	}
16803	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16804	return rcStrict;
16805	}
16806
16807
16808	/**
16809	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16810	*/
16811	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16812	{
16813	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16814
16815	/*
16816	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16817	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16818	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16819	*/
16820	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16821	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16822	\| CPUMCTX_EXTRN_HWVIRT
16823	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16824	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16825	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16826	AssertRCReturn(rc, rc);
16827
16828	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16829
16830	VMXVEXITINFO ExitInfo;
16831	RT_ZERO(ExitInfo);
16832	ExitInfo.uReason = pVmxTransient->uExitReason;
16833	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16834	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16835	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16836	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16837	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16838
16839	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16840	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16841	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16842	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16843	{
16844	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16845	rcStrict = VINF_SUCCESS;
16846	}
16847	return rcStrict;
16848	}
16849
16850
16851	/**
16852	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16853	*/
16854	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16855	{
16856	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16857
16858	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16859	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16860	\| CPUMCTX_EXTRN_HWVIRT
16861	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16862	AssertRCReturn(rc, rc);
16863
16864	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16865
16866	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16867	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16868	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16869	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16870	{
16871	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16872	rcStrict = VINF_SUCCESS;
16873	}
16874	return rcStrict;
16875	}
16876
16877
16878	/**
16879	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16880	*/
16881	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16882	{
16883	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16884
16885	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16886	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16887	\| CPUMCTX_EXTRN_HWVIRT
16888	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16889	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16890	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16891	AssertRCReturn(rc, rc);
16892
16893	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16894
16895	VMXVEXITINFO ExitInfo;
16896	RT_ZERO(ExitInfo);
16897	ExitInfo.uReason = pVmxTransient->uExitReason;
16898	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16899	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16900	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16901	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16902
16903	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16904	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16905	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16906	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16907	{
16908	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16909	rcStrict = VINF_SUCCESS;
16910	}
16911	return rcStrict;
16912	}
16913
16914
16915	/**
16916	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16917	*/
16918	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16919	{
16920	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16921
16922	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16923	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16924	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16925	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16926	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16927	AssertRCReturn(rc, rc);
16928
16929	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16930
16931	VMXVEXITINFO ExitInfo;
16932	RT_ZERO(ExitInfo);
16933	ExitInfo.uReason = pVmxTransient->uExitReason;
16934	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16935	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16936	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16937	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16938
16939	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16940	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16941	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16942	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16943	{
16944	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16945	rcStrict = VINF_SUCCESS;
16946	}
16947	return rcStrict;
16948	}
16949	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16950	/** @} */
16951
16952
16953	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16954	/** @name Nested-guest VM-exit handlers.
16955	* @{
16956	*/
16957	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16958	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16959	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16960
16961	/**
16962	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16963	* Conditional VM-exit.
16964	*/
16965	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16966	{
16967	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16968
16969	int rc = hmR0VmxCheckExitDueToEventDeliveryNested(pVCpu, pVmxTransient);
16970	AssertRCReturn(rc, rc);
16971
16972	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16973	AssertRCReturn(rc, rc);
16974
16975	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16976	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16977	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16978
16979	switch (uExitIntType)
16980	{
16981	/*
16982	* Physical NMIs:
16983	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16984	*/
16985	case VMX_EXIT_INT_INFO_TYPE_NMI:
16986	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16987
16988	/*
16989	* Hardware exceptions,
16990	* Software exceptions,
16991	* Privileged software exceptions:
16992	* Figure out if the exception must be delivered to the guest or the nested-guest.
16993	*
16994	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16995	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16996	* length. However, if delivery of a software interrupt, software exception or privileged
16997	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16998	* Hence, we read it for all exception types below to keep it simple.
16999	*/
17000	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
17001	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
17002	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
17003	{
17004	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
17005	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17006	AssertRCReturn(rc, rc);
17007
17008	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
17009	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
17010
17011	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
17012	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
17013	pVmxTransient->uExitIntErrorCode);
17014	if (fIntercept)
17015	{
17016	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17017	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17018	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17019	AssertRCReturn(rc, rc);
17020
17021	VMXVEXITINFO ExitInfo;
17022	RT_ZERO(ExitInfo);
17023	ExitInfo.uReason = pVmxTransient->uExitReason;
17024	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17025	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17026
17027	VMXVEXITEVENTINFO ExitEventInfo;
17028	RT_ZERO(ExitEventInfo);
17029	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
17030	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
17031	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17032	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17033	if (pVCpu->hm.s.Event.fPending)
17034	{
17035	Assert(ExitEventInfo.uIdtVectoringInfo == pVCpu->hm.s.Event.u64IntInfo);
17036	Assert(ExitEventInfo.uIdtVectoringErrCode == pVCpu->hm.s.Event.u32ErrCode);
17037	pVCpu->hm.s.Event.fPending = false;
17038	}
17039	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
17040	}
17041
17042	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
17043	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
17044
17045	/*
17046	* If the guest hypervisor is not intercepting an exception that caused a VM-exit directly,
17047	* forward it to the guest (for e.g, an instruction raises a #GP that causes a VM-exit but
17048	* the guest hypervisor is not intercept #GPs, inject #GP into the guest).
17049	*/
17050	if (!pVCpu->hm.s.Event.fPending)
17051	{
17052	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
17053	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
17054	}
17055	return VINF_SUCCESS;
17056	}
17057
17058	/*
17059	* Software interrupts:
17060	* VM-exits cannot be caused by software interrupts.
17061	*
17062	* External interrupts:
17063	* This should only happen when "acknowledge external interrupts on VM-exit"
17064	* control is set. However, we never set this when executing a guest or
17065	* nested-guest. For nested-guests it is emulated while injecting interrupts into
17066	* the guest.
17067	*/
17068	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
17069	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
17070	default:
17071	{
17072	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
17073	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
17074	}
17075	}
17076	}
17077
17078
17079	/**
17080	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
17081	* Unconditional VM-exit.
17082	*/
17083	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17084	{
17085	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17086	return IEMExecVmxVmexitTripleFault(pVCpu);
17087	}
17088
17089
17090	/**
17091	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
17092	*/
17093	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17094	{
17095	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17096
17097	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
17098	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17099	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
17100	}
17101
17102
17103	/**
17104	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
17105	*/
17106	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17107	{
17108	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17109
17110	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
17111	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17112	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
17113	}
17114
17115
17116	/**
17117	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
17118	* Unconditional VM-exit.
17119	*/
17120	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17121	{
17122	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17123
17124	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17125	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17126	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17127	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17128	AssertRCReturn(rc, rc);
17129
17130	VMXVEXITINFO ExitInfo;
17131	RT_ZERO(ExitInfo);
17132	ExitInfo.uReason = pVmxTransient->uExitReason;
17133	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17134	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17135
17136	VMXVEXITEVENTINFO ExitEventInfo;
17137	RT_ZERO(ExitEventInfo);
17138	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17139	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17140	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
17141	}
17142
17143
17144	/**
17145	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
17146	*/
17147	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17148	{
17149	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17150
17151	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
17152	{
17153	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17154	AssertRCReturn(rc, rc);
17155	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17156	}
17157	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
17158	}
17159
17160
17161	/**
17162	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
17163	*/
17164	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17165	{
17166	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17167
17168	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17169	{
17170	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17171	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17172	AssertRCReturn(rc, rc);
17173
17174	VMXVEXITINFO ExitInfo;
17175	RT_ZERO(ExitInfo);
17176	ExitInfo.uReason = pVmxTransient->uExitReason;
17177	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17178	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17179	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17180	}
17181	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
17182	}
17183
17184
17185	/**
17186	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
17187	*/
17188	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17189	{
17190	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17191
17192	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
17193	{
17194	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17195	AssertRCReturn(rc, rc);
17196	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17197	}
17198	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
17199	}
17200
17201
17202	/**
17203	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
17204	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
17205	*/
17206	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17207	{
17208	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17209
17210	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
17211	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
17212
17213	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17214	AssertRCReturn(rc, rc);
17215
17216	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
17217	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17218	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17219
17220	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
17221	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
17222	u64VmcsField &= UINT64_C(0xffffffff);
17223
17224	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
17225	{
17226	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17227	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17228	AssertRCReturn(rc, rc);
17229
17230	VMXVEXITINFO ExitInfo;
17231	RT_ZERO(ExitInfo);
17232	ExitInfo.uReason = pVmxTransient->uExitReason;
17233	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17234	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17235	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17236	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17237	}
17238
17239	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
17240	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
17241	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
17242	}
17243
17244
17245	/**
17246	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
17247	*/
17248	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17249	{
17250	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17251
17252	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17253	{
17254	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17255	AssertRCReturn(rc, rc);
17256	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17257	}
17258
17259	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
17260	}
17261
17262
17263	/**
17264	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
17265	* Conditional VM-exit.
17266	*/
17267	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17268	{
17269	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17270
17271	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17272	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17273	AssertRCReturn(rc, rc);
17274
17275	VBOXSTRICTRC rcStrict;
17276	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
17277	switch (uAccessType)
17278	{
17279	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
17280	{
17281	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17282	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17283	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17284	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17285
17286	bool fIntercept;
17287	switch (iCrReg)
17288	{
17289	case 0:
17290	case 4:
17291	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
17292	break;
17293
17294	case 3:
17295	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
17296	break;
17297
17298	case 8:
17299	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
17300	break;
17301
17302	default:
17303	fIntercept = false;
17304	break;
17305	}
17306	if (fIntercept)
17307	{
17308	VMXVEXITINFO ExitInfo;
17309	RT_ZERO(ExitInfo);
17310	ExitInfo.uReason = pVmxTransient->uExitReason;
17311	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17312	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17313	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17314	}
17315	else
17316	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17317	break;
17318	}
17319
17320	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
17321	{
17322	/*
17323	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
17324	* CR2 reads do not cause a VM-exit.
17325	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
17326	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
17327	*/
17328	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17329	if ( iCrReg == 3
17330	\|\| iCrReg == 8)
17331	{
17332	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
17333	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
17334	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
17335	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
17336	{
17337	VMXVEXITINFO ExitInfo;
17338	RT_ZERO(ExitInfo);
17339	ExitInfo.uReason = pVmxTransient->uExitReason;
17340	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17341	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17342	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17343	}
17344	else
17345	{
17346	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17347	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17348	}
17349	}
17350	else
17351	{
17352	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
17353	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
17354	}
17355	break;
17356	}
17357
17358	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
17359	{
17360	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
17361	Assert(pVmcsNstGst);
17362	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
17363	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
17364	if ( (uGstHostMask & X86_CR0_TS)
17365	&& (uReadShadow & X86_CR0_TS))
17366	{
17367	VMXVEXITINFO ExitInfo;
17368	RT_ZERO(ExitInfo);
17369	ExitInfo.uReason = pVmxTransient->uExitReason;
17370	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17371	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17372	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17373	}
17374	else
17375	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
17376	break;
17377	}
17378
17379	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
17380	{
17381	RTGCPTR GCPtrEffDst;
17382	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
17383	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
17384	if (fMemOperand)
17385	{
17386	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17387	AssertRCReturn(rc, rc);
17388	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
17389	}
17390	else
17391	GCPtrEffDst = NIL_RTGCPTR;
17392
17393	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
17394	{
17395	VMXVEXITINFO ExitInfo;
17396	RT_ZERO(ExitInfo);
17397	ExitInfo.uReason = pVmxTransient->uExitReason;
17398	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17399	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
17400	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17401	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17402	}
17403	else
17404	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
17405	break;
17406	}
17407
17408	default:
17409	{
17410	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
17411	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
17412	}
17413	}
17414
17415	if (rcStrict == VINF_IEM_RAISED_XCPT)
17416	{
17417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
17418	rcStrict = VINF_SUCCESS;
17419	}
17420	return rcStrict;
17421	}
17422
17423
17424	/**
17425	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
17426	* Conditional VM-exit.
17427	*/
17428	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17429	{
17430	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17431
17432	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
17433	{
17434	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17435	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17436	AssertRCReturn(rc, rc);
17437
17438	VMXVEXITINFO ExitInfo;
17439	RT_ZERO(ExitInfo);
17440	ExitInfo.uReason = pVmxTransient->uExitReason;
17441	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17442	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17443	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17444	}
17445	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
17446	}
17447
17448
17449	/**
17450	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
17451	* Conditional VM-exit.
17452	*/
17453	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17454	{
17455	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17456
17457	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17458	AssertRCReturn(rc, rc);
17459
17460	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
17461	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
17462	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
17463
17464	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
17465	uint8_t const cbAccess = s_aIOSizes[uIOSize];
17466	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
17467	{
17468	/*
17469	* IN/OUT instruction:
17470	* - Provides VM-exit instruction length.
17471	*
17472	* INS/OUTS instruction:
17473	* - Provides VM-exit instruction length.
17474	* - Provides Guest-linear address.
17475	* - Optionally provides VM-exit instruction info (depends on CPU feature).
17476	*/
17477	PVM pVM = pVCpu->CTX_SUFF(pVM);
17478	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17479	AssertRCReturn(rc, rc);
17480
17481	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17482	pVmxTransient->ExitInstrInfo.u = 0;
17483	pVmxTransient->uGuestLinearAddr = 0;
17484
17485	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17486	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17487	if (fIOString)
17488	{
17489	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17490	if (fVmxInsOutsInfo)
17491	{
17492	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17493	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17494	}
17495	}
17496	AssertRCReturn(rc, rc);
17497
17498	VMXVEXITINFO ExitInfo;
17499	RT_ZERO(ExitInfo);
17500	ExitInfo.uReason = pVmxTransient->uExitReason;
17501	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17502	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17503	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17504	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17505	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17506	}
17507	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17508	}
17509
17510
17511	/**
17512	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17513	*/
17514	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17515	{
17516	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17517
17518	uint32_t fMsrpm;
17519	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17520	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17521	else
17522	fMsrpm = VMXMSRPM_EXIT_RD;
17523
17524	if (fMsrpm & VMXMSRPM_EXIT_RD)
17525	{
17526	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17527	AssertRCReturn(rc, rc);
17528	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17529	}
17530	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17531	}
17532
17533
17534	/**
17535	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17536	*/
17537	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17538	{
17539	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17540
17541	uint32_t fMsrpm;
17542	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17543	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17544	else
17545	fMsrpm = VMXMSRPM_EXIT_WR;
17546
17547	if (fMsrpm & VMXMSRPM_EXIT_WR)
17548	{
17549	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17550	AssertRCReturn(rc, rc);
17551	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17552	}
17553	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17554	}
17555
17556
17557	/**
17558	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17559	*/
17560	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17561	{
17562	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17563
17564	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17565	{
17566	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17567	AssertRCReturn(rc, rc);
17568	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17569	}
17570	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17571	}
17572
17573
17574	/**
17575	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17576	* VM-exit.
17577	*/
17578	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17579	{
17580	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17581
17582	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17583	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17584	}
17585
17586
17587	/**
17588	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17589	*/
17590	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17591	{
17592	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17593
17594	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17595	{
17596	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17597	AssertRCReturn(rc, rc);
17598	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17599	}
17600	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17601	}
17602
17603
17604	/**
17605	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17606	*/
17607	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17608	{
17609	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17610
17611	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17612	* PAUSE when executing a nested-guest? If it does not, we would not need
17613	* to check for the intercepts here. Just call VM-exit... */
17614
17615	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17616	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17617	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17618	{
17619	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17620	AssertRCReturn(rc, rc);
17621	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17622	}
17623	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17624	}
17625
17626
17627	/**
17628	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17629	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17630	*/
17631	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17632	{
17633	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17634
17635	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17636	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17637	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17638	}
17639
17640
17641	/**
17642	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17643	* VM-exit.
17644	*/
17645	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17646	{
17647	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17648
17649	int rc = hmR0VmxCheckExitDueToEventDeliveryNested(pVCpu, pVmxTransient);
17650	AssertRCReturn(rc, rc);
17651
17652	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17653	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17654	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17655	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17656	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17657	AssertRCReturn(rc, rc);
17658
17659	VMXVEXITINFO ExitInfo;
17660	RT_ZERO(ExitInfo);
17661	ExitInfo.uReason = pVmxTransient->uExitReason;
17662	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17663	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17664
17665	VMXVEXITEVENTINFO ExitEventInfo;
17666	RT_ZERO(ExitEventInfo);
17667	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17668	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17669	if (pVCpu->hm.s.Event.fPending)
17670	{
17671	Assert(ExitEventInfo.uIdtVectoringInfo == pVCpu->hm.s.Event.u64IntInfo);
17672	Assert(ExitEventInfo.uIdtVectoringErrCode == pVCpu->hm.s.Event.u32ErrCode);
17673	pVCpu->hm.s.Event.fPending = false;
17674	}
17675	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17676	}
17677
17678
17679	/**
17680	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17681	* Conditional VM-exit.
17682	*/
17683	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17684	{
17685	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17686
17687	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17688	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17689	AssertRCReturn(rc, rc);
17690
17691	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17692	}
17693
17694
17695	/**
17696	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17697	* Conditional VM-exit.
17698	*/
17699	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17700	{
17701	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17702
17703	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17704	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17705	AssertRCReturn(rc, rc);
17706
17707	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17708	}
17709
17710
17711	/**
17712	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17713	*/
17714	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17715	{
17716	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17717
17718	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17719	{
17720	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17721	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17722	AssertRCReturn(rc, rc);
17723	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17724	}
17725	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17726	}
17727
17728
17729	/**
17730	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17731	*/
17732	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17733	{
17734	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17735
17736	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17737	{
17738	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17739	AssertRCReturn(rc, rc);
17740	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17741	}
17742	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17743	}
17744
17745
17746	/**
17747	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17748	*/
17749	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17750	{
17751	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17752
17753	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17754	{
17755	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17756	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17757	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17758	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17759	AssertRCReturn(rc, rc);
17760
17761	VMXVEXITINFO ExitInfo;
17762	RT_ZERO(ExitInfo);
17763	ExitInfo.uReason = pVmxTransient->uExitReason;
17764	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17765	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17766	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17767	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17768	}
17769	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17770	}
17771
17772
17773	/**
17774	* Nested-guest VM-exit handler for invalid-guest state
17775	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17776	*/
17777	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17778	{
17779	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17780
17781	/*
17782	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17783	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17784	* Handle it like it's in an invalid guest state of the outer guest.
17785	*
17786	* When the fast path is implemented, this should be changed to cause the corresponding
17787	* nested-guest VM-exit.
17788	*/
17789	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17790	}
17791
17792
17793	/**
17794	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17795	* and only provide the instruction length.
17796	*
17797	* Unconditional VM-exit.
17798	*/
17799	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17800	{
17801	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17802
17803	#ifdef VBOX_STRICT
17804	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17805	switch (pVmxTransient->uExitReason)
17806	{
17807	case VMX_EXIT_ENCLS:
17808	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17809	break;
17810
17811	case VMX_EXIT_VMFUNC:
17812	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17813	break;
17814	}
17815	#endif
17816
17817	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17818	AssertRCReturn(rc, rc);
17819	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17820	}
17821
17822
17823	/**
17824	* Nested-guest VM-exit handler for instructions that provide instruction length as
17825	* well as more information.
17826	*
17827	* Unconditional VM-exit.
17828	*/
17829	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17830	{
17831	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17832
17833	#ifdef VBOX_STRICT
17834	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17835	switch (pVmxTransient->uExitReason)
17836	{
17837	case VMX_EXIT_GDTR_IDTR_ACCESS:
17838	case VMX_EXIT_LDTR_TR_ACCESS:
17839	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17840	break;
17841
17842	case VMX_EXIT_RDRAND:
17843	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17844	break;
17845
17846	case VMX_EXIT_RDSEED:
17847	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17848	break;
17849
17850	case VMX_EXIT_XSAVES:
17851	case VMX_EXIT_XRSTORS:
17852	/** @todo NSTVMX: Verify XSS-bitmap. */
17853	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17854	break;
17855
17856	case VMX_EXIT_UMWAIT:
17857	case VMX_EXIT_TPAUSE:
17858	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17859	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17860	break;
17861	}
17862	#endif
17863
17864	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17865	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17866	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17867	AssertRCReturn(rc, rc);
17868
17869	VMXVEXITINFO ExitInfo;
17870	RT_ZERO(ExitInfo);
17871	ExitInfo.uReason = pVmxTransient->uExitReason;
17872	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17873	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17874	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17875	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17876	}
17877
17878	/** @} */
17879	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17880

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79698

Download in other formats: