HMVMXR0.cpp@ 79601

Last change on this file since 79601 was 79601, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Move clearing of fInterceptEvents to after we have committed to executing the nested-guest. Added HMVMX_LOG_EXIT since removing auto-load/store MSR on VM-exit happens to log stuff prior to the previous place where we logged VM-exit.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 733.0 KB

Line
1	/* $Id: HMVMXR0.cpp 79601 2019-07-08 16:21:12Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#ifdef VBOX_WITH_REM
39	# include <VBox/vmm/rem.h>
40	#endif
41	#include "HMInternal.h"
42	#include <VBox/vmm/vm.h>
43	#include <VBox/vmm/hmvmxinline.h>
44	#include "HMVMXR0.h"
45	#include "dtrace/VBoxVMM.h"
46
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
52	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
53	# define HMVMX_ALWAYS_TRAP_PF
54	# define HMVMX_ALWAYS_FLUSH_TLB
55	# define HMVMX_ALWAYS_SWAP_EFER
56	#endif
57
58
59	/*********************************************************************************************************************************
60	* Defined Constants And Macros *
61	*********************************************************************************************************************************/
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name HMVMX_READ_XXX
72	* Flags to skip redundant reads of some common VMCS fields that are not part of
73	* the guest-CPU or VCPU state but are needed while handling VM-exits.
74	*/
75	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
76	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
77	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
78	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
79	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
80	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
81	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
82	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
83	/** @} */
84
85	/**
86	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
87	* guest using hardware-assisted VMX.
88	*
89	* This excludes state like GPRs (other than RSP) which are always are
90	* swapped and restored across the world-switch and also registers like EFER,
91	* MSR which cannot be modified by the guest without causing a VM-exit.
92	*/
93	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
94	\| CPUMCTX_EXTRN_RFLAGS \
95	\| CPUMCTX_EXTRN_RSP \
96	\| CPUMCTX_EXTRN_SREG_MASK \
97	\| CPUMCTX_EXTRN_TABLE_MASK \
98	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
99	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
100	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
101	\| CPUMCTX_EXTRN_TSC_AUX \
102	\| CPUMCTX_EXTRN_OTHER_MSRS \
103	\| CPUMCTX_EXTRN_CR0 \
104	\| CPUMCTX_EXTRN_CR3 \
105	\| CPUMCTX_EXTRN_CR4 \
106	\| CPUMCTX_EXTRN_DR7 \
107	\| CPUMCTX_EXTRN_HWVIRT \
108	\| CPUMCTX_EXTRN_HM_VMX_MASK)
109
110	/**
111	* Exception bitmap mask for real-mode guests (real-on-v86).
112	*
113	* We need to intercept all exceptions manually except:
114	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
115	* due to bugs in Intel CPUs.
116	* - \#PF need not be intercepted even in real-mode if we have nested paging
117	* support.
118	*/
119	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
120	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
121	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
122	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
123	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
124	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
125	\| RT_BIT(X86_XCPT_XF))
126
127	/** Maximum VM-instruction error number. */
128	#define HMVMX_INSTR_ERROR_MAX 28
129
130	/** Profiling macro. */
131	#ifdef HM_PROFILE_EXIT_DISPATCH
132	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
133	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
134	#else
135	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
136	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
137	#endif
138
139	/** Assert that preemption is disabled or covered by thread-context hooks. */
140	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
141	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
142
143	/** Assert that we haven't migrated CPUs when thread-context hooks are not
144	* used. */
145	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
146	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
147	("Illegal migration! Entered on CPU %u Current %u\n", \
148	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
149
150	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
151	* context. */
152	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
153	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
154	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
155
156	/** Log the VM-exit reason with an easily visible marker to identify it in a
157	* potential sea of logging data. */
158	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
159	do { \
160	Log4(("VM-exit: vcpu[%RU32] reason=%#x -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", \
161	(a_pVCpu)->idCpu, (a_uExitReason))); \
162	} while (0) \
163
164
165	/*********************************************************************************************************************************
166	* Structures and Typedefs *
167	*********************************************************************************************************************************/
168	/**
169	* VMX per-VCPU transient state.
170	*
171	* A state structure for holding miscellaneous information across
172	* VMX non-root operation and restored after the transition.
173	*/
174	typedef struct VMXTRANSIENT
175	{
176	/** The host's rflags/eflags. */
177	RTCCUINTREG fEFlags;
178	#if HC_ARCH_BITS == 32
179	uint32_t u32Alignment0;
180	#endif
181	/** The guest's TPR value used for TPR shadowing. */
182	uint8_t u8GuestTpr;
183	/** Alignment. */
184	uint8_t abAlignment0[7];
185
186	/** The basic VM-exit reason. */
187	uint16_t uExitReason;
188	/** Alignment. */
189	uint16_t u16Alignment0;
190	/** The VM-exit interruption error code. */
191	uint32_t uExitIntErrorCode;
192	/** The VM-exit exit code qualification. */
193	uint64_t uExitQual;
194	/** The Guest-linear address. */
195	uint64_t uGuestLinearAddr;
196
197	/** The VM-exit interruption-information field. */
198	uint32_t uExitIntInfo;
199	/** The VM-exit instruction-length field. */
200	uint32_t cbInstr;
201	/** The VM-exit instruction-information field. */
202	VMXEXITINSTRINFO ExitInstrInfo;
203	/** Whether the VM-entry failed or not. */
204	bool fVMEntryFailed;
205	/** Whether we are currently executing a nested-guest. */
206	bool fIsNestedGuest;
207	/** Alignment. */
208	uint8_t abAlignment1[2];
209
210	/** The VM-entry interruption-information field. */
211	uint32_t uEntryIntInfo;
212	/** The VM-entry exception error code field. */
213	uint32_t uEntryXcptErrorCode;
214	/** The VM-entry instruction length field. */
215	uint32_t cbEntryInstr;
216
217	/** IDT-vectoring information field. */
218	uint32_t uIdtVectoringInfo;
219	/** IDT-vectoring error code. */
220	uint32_t uIdtVectoringErrorCode;
221
222	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
223	uint32_t fVmcsFieldsRead;
224
225	/** Whether the guest debug state was active at the time of VM-exit. */
226	bool fWasGuestDebugStateActive;
227	/** Whether the hyper debug state was active at the time of VM-exit. */
228	bool fWasHyperDebugStateActive;
229	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
230	bool fUpdatedTscOffsettingAndPreemptTimer;
231	/** Whether the VM-exit was caused by a page-fault during delivery of a
232	* contributory exception or a page-fault. */
233	bool fVectoringDoublePF;
234	/** Whether the VM-exit was caused by a page-fault during delivery of an
235	* external interrupt or NMI. */
236	bool fVectoringPF;
237	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
238	* area after VM-exit. */
239	bool fRemoveTscAuxMsr;
240	bool afAlignment0[2];
241
242	/** The VMCS info. object. */
243	PVMXVMCSINFO pVmcsInfo;
244	} VMXTRANSIENT;
245	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
246	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
247	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
248	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
249	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
250	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
251	/** Pointer to VMX transient state. */
252	typedef VMXTRANSIENT *PVMXTRANSIENT;
253	/** Pointer to a const VMX transient state. */
254	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
255
256	/**
257	* Memory operand read or write access.
258	*/
259	typedef enum VMXMEMACCESS
260	{
261	VMXMEMACCESS_READ = 0,
262	VMXMEMACCESS_WRITE = 1
263	} VMXMEMACCESS;
264
265	/**
266	* VMX VM-exit handler.
267	*
268	* @returns Strict VBox status code (i.e. informational status codes too).
269	* @param pVCpu The cross context virtual CPU structure.
270	* @param pVmxTransient The VMX-transient structure.
271	*/
272	#ifndef HMVMX_USE_FUNCTION_TABLE
273	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
274	#else
275	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
276	/** Pointer to VM-exit handler. */
277	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
278	#endif
279
280	/**
281	* VMX VM-exit handler, non-strict status code.
282	*
283	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
284	*
285	* @returns VBox status code, no informational status code returned.
286	* @param pVCpu The cross context virtual CPU structure.
287	* @param pVmxTransient The VMX-transient structure.
288	*
289	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
290	* use of that status code will be replaced with VINF_EM_SOMETHING
291	* later when switching over to IEM.
292	*/
293	#ifndef HMVMX_USE_FUNCTION_TABLE
294	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
295	#else
296	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
297	#endif
298
299
300	/*********************************************************************************************************************************
301	* Internal Functions *
302	*********************************************************************************************************************************/
303	#ifndef HMVMX_USE_FUNCTION_TABLE
304	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
305	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
306	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
307	#else
308	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
309	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
310	#endif
311	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
312	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
313	#endif
314
315	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
316	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
317	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
318	#endif
319
320	/** @name VM-exit handler prototypes.
321	* @{
322	*/
323	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
324	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
325	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
326	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
327	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
328	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
329	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
330	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
331	static FNVMXEXITHANDLER hmR0VmxExitHlt;
332	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
333	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
334	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
335	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
336	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
337	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
338	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
339	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
340	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
341	static FNVMXEXITHANDLER hmR0VmxExitVmread;
342	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
343	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
344	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
345	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
346	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
347	#endif
348	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
349	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
350	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
351	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
352	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
353	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
354	static FNVMXEXITHANDLER hmR0VmxExitMwait;
355	static FNVMXEXITHANDLER hmR0VmxExitMtf;
356	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
357	static FNVMXEXITHANDLER hmR0VmxExitPause;
358	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
359	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
360	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
361	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
362	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
363	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
365	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
366	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
370	/** @} */
371
372	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
373	/** @name Nested-guest VM-exit handler prototypes.
374	* @{
375	*/
376	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
377	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
378	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
379	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
380	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
381	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
382	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
383	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
384	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
385	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
386	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
387	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
388	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
389	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
390	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
391	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
392	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
393	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
394	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
395	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
396	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
397	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
398	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
399	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
401	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
404	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
405	/** @} */
406	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
407
408
409	/*********************************************************************************************************************************
410	* Global Variables *
411	*********************************************************************************************************************************/
412	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
413	/**
414	* Array of all VMCS fields.
415	* Any fields added to the VT-x spec. should be added here.
416	*
417	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
418	* of nested-guests.
419	*/
420	static const uint32_t g_aVmcsFields[] =
421	{
422	/* 16-bit control fields. */
423	VMX_VMCS16_VPID,
424	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
425	VMX_VMCS16_EPTP_INDEX,
426
427	/* 16-bit guest-state fields. */
428	VMX_VMCS16_GUEST_ES_SEL,
429	VMX_VMCS16_GUEST_CS_SEL,
430	VMX_VMCS16_GUEST_SS_SEL,
431	VMX_VMCS16_GUEST_DS_SEL,
432	VMX_VMCS16_GUEST_FS_SEL,
433	VMX_VMCS16_GUEST_GS_SEL,
434	VMX_VMCS16_GUEST_LDTR_SEL,
435	VMX_VMCS16_GUEST_TR_SEL,
436	VMX_VMCS16_GUEST_INTR_STATUS,
437	VMX_VMCS16_GUEST_PML_INDEX,
438
439	/* 16-bits host-state fields. */
440	VMX_VMCS16_HOST_ES_SEL,
441	VMX_VMCS16_HOST_CS_SEL,
442	VMX_VMCS16_HOST_SS_SEL,
443	VMX_VMCS16_HOST_DS_SEL,
444	VMX_VMCS16_HOST_FS_SEL,
445	VMX_VMCS16_HOST_GS_SEL,
446	VMX_VMCS16_HOST_TR_SEL,
447
448	/* 64-bit control fields. */
449	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
450	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
451	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
452	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
453	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
454	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
455	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
456	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
457	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
458	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
459	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
460	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
461	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
462	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
463	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
464	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
465	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
466	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
467	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
468	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
469	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
470	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
471	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
472	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
473	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
474	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
475	VMX_VMCS64_CTRL_EPTP_FULL,
476	VMX_VMCS64_CTRL_EPTP_HIGH,
477	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
478	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
479	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
480	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
481	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
482	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
483	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
484	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
485	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
486	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
487	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
488	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
489	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
490	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
491	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
492	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
493	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
494	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
495	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
496	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
497	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
498	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
499
500	/* 64-bit read-only data fields. */
501	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
502	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
503
504	/* 64-bit guest-state fields. */
505	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
506	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
507	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
508	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
509	VMX_VMCS64_GUEST_PAT_FULL,
510	VMX_VMCS64_GUEST_PAT_HIGH,
511	VMX_VMCS64_GUEST_EFER_FULL,
512	VMX_VMCS64_GUEST_EFER_HIGH,
513	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
514	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
515	VMX_VMCS64_GUEST_PDPTE0_FULL,
516	VMX_VMCS64_GUEST_PDPTE0_HIGH,
517	VMX_VMCS64_GUEST_PDPTE1_FULL,
518	VMX_VMCS64_GUEST_PDPTE1_HIGH,
519	VMX_VMCS64_GUEST_PDPTE2_FULL,
520	VMX_VMCS64_GUEST_PDPTE2_HIGH,
521	VMX_VMCS64_GUEST_PDPTE3_FULL,
522	VMX_VMCS64_GUEST_PDPTE3_HIGH,
523	VMX_VMCS64_GUEST_BNDCFGS_FULL,
524	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
525
526	/* 64-bit host-state fields. */
527	VMX_VMCS64_HOST_PAT_FULL,
528	VMX_VMCS64_HOST_PAT_HIGH,
529	VMX_VMCS64_HOST_EFER_FULL,
530	VMX_VMCS64_HOST_EFER_HIGH,
531	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
532	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
533
534	/* 32-bit control fields. */
535	VMX_VMCS32_CTRL_PIN_EXEC,
536	VMX_VMCS32_CTRL_PROC_EXEC,
537	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
538	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
539	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
540	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
541	VMX_VMCS32_CTRL_EXIT,
542	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
543	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
544	VMX_VMCS32_CTRL_ENTRY,
545	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
546	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
547	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
548	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
549	VMX_VMCS32_CTRL_TPR_THRESHOLD,
550	VMX_VMCS32_CTRL_PROC_EXEC2,
551	VMX_VMCS32_CTRL_PLE_GAP,
552	VMX_VMCS32_CTRL_PLE_WINDOW,
553
554	/* 32-bits read-only fields. */
555	VMX_VMCS32_RO_VM_INSTR_ERROR,
556	VMX_VMCS32_RO_EXIT_REASON,
557	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
558	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
559	VMX_VMCS32_RO_IDT_VECTORING_INFO,
560	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
561	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
562	VMX_VMCS32_RO_EXIT_INSTR_INFO,
563
564	/* 32-bit guest-state fields. */
565	VMX_VMCS32_GUEST_ES_LIMIT,
566	VMX_VMCS32_GUEST_CS_LIMIT,
567	VMX_VMCS32_GUEST_SS_LIMIT,
568	VMX_VMCS32_GUEST_DS_LIMIT,
569	VMX_VMCS32_GUEST_FS_LIMIT,
570	VMX_VMCS32_GUEST_GS_LIMIT,
571	VMX_VMCS32_GUEST_LDTR_LIMIT,
572	VMX_VMCS32_GUEST_TR_LIMIT,
573	VMX_VMCS32_GUEST_GDTR_LIMIT,
574	VMX_VMCS32_GUEST_IDTR_LIMIT,
575	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
576	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
577	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
578	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
579	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
580	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
581	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
582	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
583	VMX_VMCS32_GUEST_INT_STATE,
584	VMX_VMCS32_GUEST_ACTIVITY_STATE,
585	VMX_VMCS32_GUEST_SMBASE,
586	VMX_VMCS32_GUEST_SYSENTER_CS,
587	VMX_VMCS32_PREEMPT_TIMER_VALUE,
588
589	/* 32-bit host-state fields. */
590	VMX_VMCS32_HOST_SYSENTER_CS,
591
592	/* Natural-width control fields. */
593	VMX_VMCS_CTRL_CR0_MASK,
594	VMX_VMCS_CTRL_CR4_MASK,
595	VMX_VMCS_CTRL_CR0_READ_SHADOW,
596	VMX_VMCS_CTRL_CR4_READ_SHADOW,
597	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
598	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
599	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
600	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
601
602	/* Natural-width read-only data fields. */
603	VMX_VMCS_RO_EXIT_QUALIFICATION,
604	VMX_VMCS_RO_IO_RCX,
605	VMX_VMCS_RO_IO_RSI,
606	VMX_VMCS_RO_IO_RDI,
607	VMX_VMCS_RO_IO_RIP,
608	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
609
610	/* Natural-width guest-state field */
611	VMX_VMCS_GUEST_CR0,
612	VMX_VMCS_GUEST_CR3,
613	VMX_VMCS_GUEST_CR4,
614	VMX_VMCS_GUEST_ES_BASE,
615	VMX_VMCS_GUEST_CS_BASE,
616	VMX_VMCS_GUEST_SS_BASE,
617	VMX_VMCS_GUEST_DS_BASE,
618	VMX_VMCS_GUEST_FS_BASE,
619	VMX_VMCS_GUEST_GS_BASE,
620	VMX_VMCS_GUEST_LDTR_BASE,
621	VMX_VMCS_GUEST_TR_BASE,
622	VMX_VMCS_GUEST_GDTR_BASE,
623	VMX_VMCS_GUEST_IDTR_BASE,
624	VMX_VMCS_GUEST_DR7,
625	VMX_VMCS_GUEST_RSP,
626	VMX_VMCS_GUEST_RIP,
627	VMX_VMCS_GUEST_RFLAGS,
628	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
629	VMX_VMCS_GUEST_SYSENTER_ESP,
630	VMX_VMCS_GUEST_SYSENTER_EIP,
631
632	/* Natural-width host-state fields */
633	VMX_VMCS_HOST_CR0,
634	VMX_VMCS_HOST_CR3,
635	VMX_VMCS_HOST_CR4,
636	VMX_VMCS_HOST_FS_BASE,
637	VMX_VMCS_HOST_GS_BASE,
638	VMX_VMCS_HOST_TR_BASE,
639	VMX_VMCS_HOST_GDTR_BASE,
640	VMX_VMCS_HOST_IDTR_BASE,
641	VMX_VMCS_HOST_SYSENTER_ESP,
642	VMX_VMCS_HOST_SYSENTER_EIP,
643	VMX_VMCS_HOST_RSP,
644	VMX_VMCS_HOST_RIP
645	};
646	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
647
648	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
649	static const uint32_t g_aVmcsCacheSegBase[] =
650	{
651	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
652	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
653	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
654	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
655	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
656	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
657	};
658	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
659	#endif
660	static const uint32_t g_aVmcsSegBase[] =
661	{
662	VMX_VMCS_GUEST_ES_BASE,
663	VMX_VMCS_GUEST_CS_BASE,
664	VMX_VMCS_GUEST_SS_BASE,
665	VMX_VMCS_GUEST_DS_BASE,
666	VMX_VMCS_GUEST_FS_BASE,
667	VMX_VMCS_GUEST_GS_BASE
668	};
669	static const uint32_t g_aVmcsSegSel[] =
670	{
671	VMX_VMCS16_GUEST_ES_SEL,
672	VMX_VMCS16_GUEST_CS_SEL,
673	VMX_VMCS16_GUEST_SS_SEL,
674	VMX_VMCS16_GUEST_DS_SEL,
675	VMX_VMCS16_GUEST_FS_SEL,
676	VMX_VMCS16_GUEST_GS_SEL
677	};
678	static const uint32_t g_aVmcsSegLimit[] =
679	{
680	VMX_VMCS32_GUEST_ES_LIMIT,
681	VMX_VMCS32_GUEST_CS_LIMIT,
682	VMX_VMCS32_GUEST_SS_LIMIT,
683	VMX_VMCS32_GUEST_DS_LIMIT,
684	VMX_VMCS32_GUEST_FS_LIMIT,
685	VMX_VMCS32_GUEST_GS_LIMIT
686	};
687	static const uint32_t g_aVmcsSegAttr[] =
688	{
689	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
690	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
691	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
692	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
693	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
694	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
695	};
696	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
697	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
698	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
699	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
700
701	#ifdef HMVMX_USE_FUNCTION_TABLE
702	/**
703	* VMX_EXIT dispatch table.
704	*/
705	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
706	{
707	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
708	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
709	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
710	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
711	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
712	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
713	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
714	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
715	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
716	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
717	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
718	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
719	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
720	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
721	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
722	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
723	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
724	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
725	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
726	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
727	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
728	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
729	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
730	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
731	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
732	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
733	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
734	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
735	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
736	#else
737	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
738	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
739	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
740	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
741	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
742	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
743	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
744	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
745	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
746	#endif
747	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
748	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
749	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
750	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
751	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
752	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
753	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
754	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
755	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
756	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
757	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
758	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
759	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
760	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
761	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
762	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
763	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
764	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
765	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
766	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
767	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
768	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
769	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
770	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
771	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
772	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
773	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
774	#else
775	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
776	#endif
777	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
778	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
779	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
780	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
781	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
782	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
783	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
784	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
785	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
786	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
787	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
788	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
789	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
790	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
791	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
792	};
793	#endif /* HMVMX_USE_FUNCTION_TABLE */
794
795	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
796	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
797	{
798	/* 0 */ "(Not Used)",
799	/* 1 */ "VMCALL executed in VMX root operation.",
800	/* 2 */ "VMCLEAR with invalid physical address.",
801	/* 3 */ "VMCLEAR with VMXON pointer.",
802	/* 4 */ "VMLAUNCH with non-clear VMCS.",
803	/* 5 */ "VMRESUME with non-launched VMCS.",
804	/* 6 */ "VMRESUME after VMXOFF",
805	/* 7 */ "VM-entry with invalid control fields.",
806	/* 8 */ "VM-entry with invalid host state fields.",
807	/* 9 */ "VMPTRLD with invalid physical address.",
808	/* 10 */ "VMPTRLD with VMXON pointer.",
809	/* 11 */ "VMPTRLD with incorrect revision identifier.",
810	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
811	/* 13 */ "VMWRITE to read-only VMCS component.",
812	/* 14 */ "(Not Used)",
813	/* 15 */ "VMXON executed in VMX root operation.",
814	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
815	/* 17 */ "VM-entry with non-launched executing VMCS.",
816	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
817	/* 19 */ "VMCALL with non-clear VMCS.",
818	/* 20 */ "VMCALL with invalid VM-exit control fields.",
819	/* 21 */ "(Not Used)",
820	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
821	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
822	/* 24 */ "VMCALL with invalid SMM-monitor features.",
823	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
824	/* 26 */ "VM-entry with events blocked by MOV SS.",
825	/* 27 */ "(Not Used)",
826	/* 28 */ "Invalid operand to INVEPT/INVVPID."
827	};
828	#endif /* VBOX_STRICT && LOG_ENABLED */
829
830
831	/**
832	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
833	*
834	* Any bit set in this mask is owned by the host/hypervisor and would cause a
835	* VM-exit when modified by the guest.
836	*
837	* @returns The static CR0 guest/host mask.
838	* @param pVCpu The cross context virtual CPU structure.
839	*/
840	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
841	{
842	/*
843	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
844	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
845	*/
846	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
847	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
848	* and @bugref{6944}. */
849	PVM pVM = pVCpu->CTX_SUFF(pVM);
850	return ( X86_CR0_PE
851	\| X86_CR0_NE
852	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
853	\| X86_CR0_PG
854	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
855	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
856	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
857	}
858
859
860	/**
861	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
862	*
863	* Any bit set in this mask is owned by the host/hypervisor and would cause a
864	* VM-exit when modified by the guest.
865	*
866	* @returns The static CR4 guest/host mask.
867	* @param pVCpu The cross context virtual CPU structure.
868	*/
869	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
870	{
871	/*
872	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
873	* these bits are reserved on hardware that does not support them. Since the
874	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
875	* these bits and handle it depending on whether we expose them to the guest.
876	*/
877	PVM pVM = pVCpu->CTX_SUFF(pVM);
878	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
879	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
880	return ( X86_CR4_VMXE
881	\| X86_CR4_VME
882	\| X86_CR4_PAE
883	\| X86_CR4_PGE
884	\| X86_CR4_PSE
885	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
886	\| (fPcid ? X86_CR4_PCIDE : 0));
887	}
888
889
890	/**
891	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
892	* area.
893	*
894	* @returns @c true if it's different, @c false otherwise.
895	* @param pVmcsInfo The VMCS info. object.
896	*/
897	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
898	{
899	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
900	&& pVmcsInfo->pvGuestMsrStore);
901	}
902
903
904	/**
905	* Checks whether one of the given Pin-based VM-execution controls are set.
906	*
907	* @returns @c true if set, @c false otherwise.
908	* @param pVCpu The cross context virtual CPU structure.
909	* @param pVmxTransient The VMX-transient structure.
910	* @param uPinCtls The Pin-based VM-execution controls to check.
911	*
912	* @remarks This will not check merged controls when executing a nested-guest
913	* but the original control specified by the guest hypervisor.
914	*/
915	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
916	{
917	if (!pVmxTransient->fIsNestedGuest)
918	{
919	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
920	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
921	}
922	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
923	}
924
925
926	#if 0
927	/**
928	* Checks whether one of the given Processor-based VM-execution controls are set.
929	*
930	* @returns @c true if set, @c false otherwise.
931	* @param pVCpu The cross context virtual CPU structure.
932	* @param pVmxTransient The VMX-transient structure.
933	* @param uProcCtls The Processor-based VM-execution controls to check.
934	*
935	* @remarks This will not check merged controls when executing a nested-guest
936	* but the original control specified by the guest hypervisor.
937	*/
938	static bool hmR0VmxIsProcCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
939	{
940	if (!pVmxTransient->fIsNestedGuest)
941	{
942	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
943	return RT_BOOL(pVmcsInfo->u32ProcCtls & uProcCtls);
944	}
945	return CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
946	}
947
948
949	/**
950	* Checks whether one of the given Secondary Processor-based VM-execution controls
951	* are set.
952	*
953	* @returns @c true if set, @c false otherwise.
954	* @param pVCpu The cross context virtual CPU structure.
955	* @param pVmxTransient The VMX-transient structure.
956	* @param uProcCtls2 The Secondary Processor-based VM-execution controls to
957	* check.
958	*
959	* @remarks This will not check merged controls when executing a nested-guest
960	* but the original control specified by the guest hypervisor.
961	*/
962	static bool hmR0VmxIsProcCtls2Set(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls2)
963	{
964	if (!pVmxTransient->fIsNestedGuest)
965	{
966	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
967	return RT_BOOL(pVmcsInfo->u32ProcCtls2 & uProcCtls2);
968	}
969	return CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls2);
970	}
971
972
973	/**
974	* Checks whether one of the given VM-entry controls are set.
975	*
976	* @returns @c true if set, @c false otherwise.
977	* @param pVCpu The cross context virtual CPU structure.
978	* @param pVmxTransient The VMX-transient structure.
979	* @param uEntryCtls The VM-entry controls to check.
980	*
981	* @remarks This will not check merged controls when executing a nested-guest
982	* but the original control specified by the guest hypervisor.
983	*/
984	static bool hmR0VmxIsEntryCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uEntryCtls)
985	{
986	if (!pVmxTransient->fIsNestedGuest)
987	{
988	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
989	return RT_BOOL(pVmcsInfo->u32EntryCtls & uEntryCtls);
990	}
991	return CPUMIsGuestVmxEntryCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uEntryCtls);
992	}
993
994
995	/**
996	* Checks whether one of the given VM-exit controls are set.
997	*
998	* @returns @c true if set, @c false otherwise.
999	* @param pVCpu The cross context virtual CPU structure.
1000	* @param pVmxTransient The VMX-transient structure.
1001	* @param uExitCtls The VM-exit controls to check.
1002	*
1003	* @remarks This will not check merged controls when executing a nested-guest
1004	* but the original control specified by the guest hypervisor.
1005	*/
1006	static bool hmR0VmxIsExitCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uExitCtls)
1007	{
1008	if (!pVmxTransient->fIsNestedGuest)
1009	{
1010	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1011	return RT_BOOL(pVmcsInfo->u32ExitCtls & uExitCtls);
1012	}
1013	return CPUMIsGuestVmxExitCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uExitCtls);
1014	}
1015	#endif
1016
1017
1018	/**
1019	* Adds one or more exceptions to the exception bitmap and commits it to the current
1020	* VMCS.
1021	*
1022	* @returns VBox status code.
1023	* @param pVmxTransient The VMX-transient structure.
1024	* @param uXcptMask The exception(s) to add.
1025	*/
1026	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1027	{
1028	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1029	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1030	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1031	{
1032	uXcptBitmap \|= uXcptMask;
1033	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1034	AssertRCReturn(rc, rc);
1035	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1036	}
1037	return VINF_SUCCESS;
1038	}
1039
1040
1041	/**
1042	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1043	*
1044	* @returns VBox status code.
1045	* @param pVmxTransient The VMX-transient structure.
1046	* @param uXcpt The exception to add.
1047	*/
1048	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1049	{
1050	Assert(uXcpt <= X86_XCPT_LAST);
1051	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1052	}
1053
1054
1055	/**
1056	* Remove one or more exceptions from the exception bitmap and commits it to the
1057	* current VMCS.
1058	*
1059	* This takes care of not removing the exception intercept if a nested-guest
1060	* requires the exception to be intercepted.
1061	*
1062	* @returns VBox status code.
1063	* @param pVCpu The cross context virtual CPU structure.
1064	* @param pVmxTransient The VMX-transient structure.
1065	* @param uXcptMask The exception(s) to remove.
1066	*/
1067	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1068	{
1069	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1070	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1071	if (u32XcptBitmap & uXcptMask)
1072	{
1073	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1074	if (!pVmxTransient->fIsNestedGuest)
1075	{ /* likely */ }
1076	else
1077	{
1078	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1079	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1080	}
1081	#endif
1082	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1083	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1084	\| RT_BIT(X86_XCPT_DE)
1085	\| RT_BIT(X86_XCPT_NM)
1086	\| RT_BIT(X86_XCPT_TS)
1087	\| RT_BIT(X86_XCPT_UD)
1088	\| RT_BIT(X86_XCPT_NP)
1089	\| RT_BIT(X86_XCPT_SS)
1090	\| RT_BIT(X86_XCPT_GP)
1091	\| RT_BIT(X86_XCPT_PF)
1092	\| RT_BIT(X86_XCPT_MF));
1093	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1094	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1095	#endif
1096	if (uXcptMask)
1097	{
1098	/* Validate we are not removing any essential exception intercepts. */
1099	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1100	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1101	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1102
1103	/* Remove it from the exception bitmap. */
1104	u32XcptBitmap &= ~uXcptMask;
1105
1106	/* Commit and update the cache if necessary. */
1107	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1108	{
1109	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1110	AssertRCReturn(rc, rc);
1111	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1112	}
1113	}
1114	}
1115	return VINF_SUCCESS;
1116	}
1117
1118
1119	/**
1120	* Remove an exceptions from the exception bitmap and commits it to the current
1121	* VMCS.
1122	*
1123	* @returns VBox status code.
1124	* @param pVCpu The cross context virtual CPU structure.
1125	* @param pVmxTransient The VMX-transient structure.
1126	* @param uXcpt The exception to remove.
1127	*/
1128	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1129	{
1130	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1131	}
1132
1133
1134	/**
1135	* Loads the VMCS specified by the VMCS info. object.
1136	*
1137	* @returns VBox status code.
1138	* @param pVmcsInfo The VMCS info. object.
1139	*
1140	* @remarks Can be called with interrupts disabled.
1141	*/
1142	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1143	{
1144	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1145	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1146
1147	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1148	if (RT_SUCCESS(rc))
1149	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1150	return rc;
1151	}
1152
1153
1154	/**
1155	* Clears the VMCS specified by the VMCS info. object.
1156	*
1157	* @returns VBox status code.
1158	* @param pVmcsInfo The VMCS info. object.
1159	*
1160	* @remarks Can be called with interrupts disabled.
1161	*/
1162	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1163	{
1164	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1165	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1166
1167	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1168	if (RT_SUCCESS(rc))
1169	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1170	return rc;
1171	}
1172
1173
1174	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1175	/**
1176	* Loads the shadow VMCS specified by the VMCS info. object.
1177	*
1178	* @returns VBox status code.
1179	* @param pVmcsInfo The VMCS info. object.
1180	*
1181	* @remarks Can be called with interrupts disabled.
1182	*/
1183	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1184	{
1185	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1186	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1187
1188	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1189	if (RT_SUCCESS(rc))
1190	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1191	return rc;
1192	}
1193
1194
1195	/**
1196	* Clears the shadow VMCS specified by the VMCS info. object.
1197	*
1198	* @returns VBox status code.
1199	* @param pVmcsInfo The VMCS info. object.
1200	*
1201	* @remarks Can be called with interrupts disabled.
1202	*/
1203	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1204	{
1205	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1206	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1207
1208	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1209	if (RT_SUCCESS(rc))
1210	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1211	return rc;
1212	}
1213
1214
1215	/**
1216	* Switches from and to the specified VMCSes.
1217	*
1218	* @returns VBox status code.
1219	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1220	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1221	*
1222	* @remarks Called with interrupts disabled.
1223	*/
1224	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1225	{
1226	/*
1227	* Clear the VMCS we are switching out if it has not already been cleared.
1228	* This will sync any CPU internal data back to the VMCS.
1229	*/
1230	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1231	{
1232	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1233	if (RT_SUCCESS(rc))
1234	{
1235	/*
1236	* The shadow VMCS, if any, would not be active at this point since we
1237	* would have cleared it while importing the virtual hardware-virtualization
1238	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1239	* clear the shadow VMCS here, just assert for safety.
1240	*/
1241	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1242	}
1243	else
1244	return rc;
1245	}
1246
1247	/*
1248	* Clear the VMCS we are switching to if it has not already been cleared.
1249	* This will initialize the VMCS launch state to "clear" required for loading it.
1250	*
1251	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1252	*/
1253	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1254	{
1255	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1256	if (RT_SUCCESS(rc))
1257	{ /* likely */ }
1258	else
1259	return rc;
1260	}
1261
1262	/*
1263	* Finally, load the VMCS we are switching to.
1264	*/
1265	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1266	}
1267
1268
1269	/**
1270	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1271	* caller.
1272	*
1273	* @returns VBox status code.
1274	* @param pVCpu The cross context virtual CPU structure.
1275	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1276	* true) or guest VMCS (pass false).
1277	*/
1278	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1279	{
1280	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1281	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1282
1283	PVMXVMCSINFO pVmcsInfoFrom;
1284	PVMXVMCSINFO pVmcsInfoTo;
1285	if (fSwitchToNstGstVmcs)
1286	{
1287	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1288	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1289	}
1290	else
1291	{
1292	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1293	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1294	}
1295
1296	/*
1297	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1298	* preemption hook code path acquires the current VMCS.
1299	*/
1300	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1301
1302	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1303	if (RT_SUCCESS(rc))
1304	{
1305	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1306
1307	/*
1308	* If we are switching to a VMCS that was executed on a different host CPU or was
1309	* never executed before, flag that we need to export the host state before executing
1310	* guest/nested-guest code using hardware-assisted VMX.
1311	*
1312	* This could probably be done in a preemptible context since the preemption hook
1313	* will flag the necessary change in host context. However, since preemption is
1314	* already disabled and to avoid making assumptions about host specific code in
1315	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1316	* disabled.
1317	*/
1318	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1319	{ /* likely */ }
1320	else
1321	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1322
1323	ASMSetFlags(fEFlags);
1324
1325	/*
1326	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1327	* flag that we need to update the host MSR values there. Even if we decide in the
1328	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1329	* if its content differs, we would have to update the host MSRs anyway.
1330	*/
1331	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1332	}
1333	else
1334	ASMSetFlags(fEFlags);
1335	return rc;
1336	}
1337	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1338
1339
1340	/**
1341	* Updates the VM's last error record.
1342	*
1343	* If there was a VMX instruction error, reads the error data from the VMCS and
1344	* updates VCPU's last error record as well.
1345	*
1346	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1347	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1348	* VERR_VMX_INVALID_VMCS_FIELD.
1349	* @param rc The error code.
1350	*/
1351	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1352	{
1353	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1354	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1355	{
1356	AssertPtrReturnVoid(pVCpu);
1357	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1358	}
1359	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1360	}
1361
1362
1363	#ifdef VBOX_STRICT
1364	/**
1365	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1366	* transient structure.
1367	*
1368	* @returns VBox status code.
1369	* @param pVmxTransient The VMX-transient structure.
1370	*
1371	* @remarks No-long-jump zone!!!
1372	*/
1373	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1374	{
1375	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1376	AssertRCReturn(rc, rc);
1377	return VINF_SUCCESS;
1378	}
1379
1380
1381	/**
1382	* Reads the VM-entry exception error code field from the VMCS into
1383	* the VMX transient structure.
1384	*
1385	* @returns VBox status code.
1386	* @param pVmxTransient The VMX-transient structure.
1387	*
1388	* @remarks No-long-jump zone!!!
1389	*/
1390	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1391	{
1392	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1393	AssertRCReturn(rc, rc);
1394	return VINF_SUCCESS;
1395	}
1396
1397
1398	/**
1399	* Reads the VM-entry exception error code field from the VMCS into
1400	* the VMX transient structure.
1401	*
1402	* @returns VBox status code.
1403	* @param pVmxTransient The VMX-transient structure.
1404	*
1405	* @remarks No-long-jump zone!!!
1406	*/
1407	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1408	{
1409	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1410	AssertRCReturn(rc, rc);
1411	return VINF_SUCCESS;
1412	}
1413	#endif /* VBOX_STRICT */
1414
1415
1416	/**
1417	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1418	* transient structure.
1419	*
1420	* @returns VBox status code.
1421	* @param pVmxTransient The VMX-transient structure.
1422	*/
1423	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1424	{
1425	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1426	{
1427	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1428	AssertRCReturn(rc,rc);
1429	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1430	}
1431	return VINF_SUCCESS;
1432	}
1433
1434
1435	/**
1436	* Reads the VM-exit interruption error code from the VMCS into the VMX
1437	* transient structure.
1438	*
1439	* @returns VBox status code.
1440	* @param pVmxTransient The VMX-transient structure.
1441	*/
1442	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1443	{
1444	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1445	{
1446	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1447	AssertRCReturn(rc, rc);
1448	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1449	}
1450	return VINF_SUCCESS;
1451	}
1452
1453
1454	/**
1455	* Reads the VM-exit instruction length field from the VMCS into the VMX
1456	* transient structure.
1457	*
1458	* @returns VBox status code.
1459	* @param pVmxTransient The VMX-transient structure.
1460	*/
1461	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1462	{
1463	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1464	{
1465	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1466	AssertRCReturn(rc, rc);
1467	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1468	}
1469	return VINF_SUCCESS;
1470	}
1471
1472
1473	/**
1474	* Reads the VM-exit instruction-information field from the VMCS into
1475	* the VMX transient structure.
1476	*
1477	* @returns VBox status code.
1478	* @param pVmxTransient The VMX-transient structure.
1479	*/
1480	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1481	{
1482	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1483	{
1484	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1485	AssertRCReturn(rc, rc);
1486	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1487	}
1488	return VINF_SUCCESS;
1489	}
1490
1491
1492	/**
1493	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1494	*
1495	* @returns VBox status code.
1496	* @param pVCpu The cross context virtual CPU structure of the
1497	* calling EMT. (Required for the VMCS cache case.)
1498	* @param pVmxTransient The VMX-transient structure.
1499	*/
1500	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1501	{
1502	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1503	{
1504	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1505	AssertRCReturn(rc, rc);
1506	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1507	}
1508	return VINF_SUCCESS;
1509	}
1510
1511
1512	/**
1513	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1514	*
1515	* @returns VBox status code.
1516	* @param pVCpu The cross context virtual CPU structure of the
1517	* calling EMT. (Required for the VMCS cache case.)
1518	* @param pVmxTransient The VMX-transient structure.
1519	*/
1520	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1521	{
1522	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1523	{
1524	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1525	AssertRCReturn(rc, rc);
1526	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1527	}
1528	return VINF_SUCCESS;
1529	}
1530
1531
1532	/**
1533	* Reads the IDT-vectoring information field from the VMCS into the VMX
1534	* transient structure.
1535	*
1536	* @returns VBox status code.
1537	* @param pVmxTransient The VMX-transient structure.
1538	*
1539	* @remarks No-long-jump zone!!!
1540	*/
1541	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1542	{
1543	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1544	{
1545	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1546	AssertRCReturn(rc, rc);
1547	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1548	}
1549	return VINF_SUCCESS;
1550	}
1551
1552
1553	/**
1554	* Reads the IDT-vectoring error code from the VMCS into the VMX
1555	* transient structure.
1556	*
1557	* @returns VBox status code.
1558	* @param pVmxTransient The VMX-transient structure.
1559	*/
1560	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1561	{
1562	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1563	{
1564	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1565	AssertRCReturn(rc, rc);
1566	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1567	}
1568	return VINF_SUCCESS;
1569	}
1570
1571
1572	/**
1573	* Enters VMX root mode operation on the current CPU.
1574	*
1575	* @returns VBox status code.
1576	* @param pVM The cross context VM structure. Can be
1577	* NULL, after a resume.
1578	* @param HCPhysCpuPage Physical address of the VMXON region.
1579	* @param pvCpuPage Pointer to the VMXON region.
1580	*/
1581	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1582	{
1583	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1584	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1585	Assert(pvCpuPage);
1586	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1587
1588	if (pVM)
1589	{
1590	/* Write the VMCS revision identifier to the VMXON region. */
1591	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1592	}
1593
1594	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1595	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1596
1597	/* Enable the VMX bit in CR4 if necessary. */
1598	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1599
1600	/* Enter VMX root mode. */
1601	int rc = VMXEnable(HCPhysCpuPage);
1602	if (RT_FAILURE(rc))
1603	{
1604	if (!(uOldCr4 & X86_CR4_VMXE))
1605	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1606
1607	if (pVM)
1608	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1609	}
1610
1611	/* Restore interrupts. */
1612	ASMSetFlags(fEFlags);
1613	return rc;
1614	}
1615
1616
1617	/**
1618	* Exits VMX root mode operation on the current CPU.
1619	*
1620	* @returns VBox status code.
1621	*/
1622	static int hmR0VmxLeaveRootMode(void)
1623	{
1624	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1625
1626	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1627	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1628
1629	/* If we're for some reason not in VMX root mode, then don't leave it. */
1630	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1631
1632	int rc;
1633	if (uHostCr4 & X86_CR4_VMXE)
1634	{
1635	/* Exit VMX root mode and clear the VMX bit in CR4. */
1636	VMXDisable();
1637	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1638	rc = VINF_SUCCESS;
1639	}
1640	else
1641	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1642
1643	/* Restore interrupts. */
1644	ASMSetFlags(fEFlags);
1645	return rc;
1646	}
1647
1648
1649	/**
1650	* Allocates and maps a physically contiguous page. The allocated page is
1651	* zero'd out (used by various VT-x structures).
1652	*
1653	* @returns IPRT status code.
1654	* @param pMemObj Pointer to the ring-0 memory object.
1655	* @param ppVirt Where to store the virtual address of the allocation.
1656	* @param pHCPhys Where to store the physical address of the allocation.
1657	*/
1658	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1659	{
1660	AssertPtr(pMemObj);
1661	AssertPtr(ppVirt);
1662	AssertPtr(pHCPhys);
1663	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1664	if (RT_FAILURE(rc))
1665	return rc;
1666	ppVirt = RTR0MemObjAddress(pMemObj);
1667	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1668	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1669	return VINF_SUCCESS;
1670	}
1671
1672
1673	/**
1674	* Frees and unmaps an allocated, physical page.
1675	*
1676	* @param pMemObj Pointer to the ring-0 memory object.
1677	* @param ppVirt Where to re-initialize the virtual address of allocation as
1678	* 0.
1679	* @param pHCPhys Where to re-initialize the physical address of the
1680	* allocation as 0.
1681	*/
1682	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1683	{
1684	AssertPtr(pMemObj);
1685	AssertPtr(ppVirt);
1686	AssertPtr(pHCPhys);
1687	/* NULL is valid, accepted and ignored by the free function below. */
1688	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1689	*pMemObj = NIL_RTR0MEMOBJ;
1690	*ppVirt = NULL;
1691	*pHCPhys = NIL_RTHCPHYS;
1692	}
1693
1694
1695	/**
1696	* Initializes a VMCS info. object.
1697	*
1698	* @param pVmcsInfo The VMCS info. object.
1699	*/
1700	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1701	{
1702	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1703
1704	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1705	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1706	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1707	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1708	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1709	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1710	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1711	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1712	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1713	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1714	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1715	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1716	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1717	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1718	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1719	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1720	}
1721
1722
1723	/**
1724	* Frees the VT-x structures for a VMCS info. object.
1725	*
1726	* @param pVM The cross context VM structure.
1727	* @param pVmcsInfo The VMCS info. object.
1728	*/
1729	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1730	{
1731	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1732
1733	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1734	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1735	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1736	#endif
1737
1738	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1739	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1740
1741	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1742	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1743	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1744
1745	hmR0VmxInitVmcsInfo(pVmcsInfo);
1746	}
1747
1748
1749	/**
1750	* Allocates the VT-x structures for a VMCS info. object.
1751	*
1752	* @returns VBox status code.
1753	* @param pVCpu The cross context virtual CPU structure.
1754	* @param pVmcsInfo The VMCS info. object.
1755	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1756	*/
1757	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1758	{
1759	PVM pVM = pVCpu->CTX_SUFF(pVM);
1760
1761	/* Allocate the guest VM control structure (VMCS). */
1762	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1763	if (RT_SUCCESS(rc))
1764	{
1765	if (!fIsNstGstVmcs)
1766	{
1767	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1768	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1769	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1770	#endif
1771	if (RT_SUCCESS(rc))
1772	{
1773	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1774	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1775	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1776	{
1777	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1778	NULL /* pR3Ptr /, NULL / pRCPtr */);
1779	}
1780	}
1781	}
1782	else
1783	{
1784	/* We don't yet support exposing VMCS shadowing to the guest. */
1785	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1786	Assert(!pVmcsInfo->pvShadowVmcs);
1787
1788	/* Get the allocated virtual-APIC page from CPUM. */
1789	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1790	{
1791	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1792	&pVmcsInfo->HCPhysVirtApic);
1793	Assert(pVmcsInfo->pbVirtApic);
1794	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1795	}
1796	}
1797
1798	if (RT_SUCCESS(rc))
1799	{
1800	/*
1801	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1802	* transparent accesses of specific MSRs.
1803	*
1804	* If the condition for enabling MSR bitmaps changes here, don't forget to
1805	* update HMIsMsrBitmapActive().
1806	*
1807	* We don't share MSR bitmaps between the guest and nested-guest as we then
1808	* don't need to care about carefully restoring the guest MSR bitmap.
1809	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1810	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1811	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1812	* we do that later while merging VMCS.
1813	*/
1814	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1815	{
1816	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1817	if ( RT_SUCCESS(rc)
1818	&& !fIsNstGstVmcs)
1819	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1820	}
1821
1822	if (RT_SUCCESS(rc))
1823	{
1824	/*
1825	* Allocate the VM-entry MSR-load area for the guest MSRs.
1826	*
1827	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1828	* the guest and nested-guest.
1829	*/
1830	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1831	&pVmcsInfo->HCPhysGuestMsrLoad);
1832	if (RT_SUCCESS(rc))
1833	{
1834	/*
1835	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1836	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1837	*/
1838	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1839	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1840	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1841
1842	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1843	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1844	&pVmcsInfo->HCPhysHostMsrLoad);
1845	}
1846	}
1847	}
1848	}
1849
1850	return rc;
1851	}
1852
1853
1854	/**
1855	* Free all VT-x structures for the VM.
1856	*
1857	* @returns IPRT status code.
1858	* @param pVM The cross context VM structure.
1859	*/
1860	static void hmR0VmxStructsFree(PVM pVM)
1861	{
1862	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1863	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1864	#endif
1865	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1866
1867	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1868	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1869	{
1870	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1871	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1872	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1873	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1874	}
1875	#endif
1876
1877	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1878	{
1879	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1880	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1881	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1882	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1883	if (pVM->cpum.ro.GuestFeatures.fVmx)
1884	{
1885	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1886	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1887	}
1888	#endif
1889	}
1890	}
1891
1892
1893	/**
1894	* Allocate all VT-x structures for the VM.
1895	*
1896	* @returns IPRT status code.
1897	* @param pVM The cross context VM structure.
1898	*/
1899	static int hmR0VmxStructsAlloc(PVM pVM)
1900	{
1901	/*
1902	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1903	* The VMCS size cannot be more than 4096 bytes.
1904	*
1905	* See Intel spec. Appendix A.1 "Basic VMX Information".
1906	*/
1907	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1908	if (cbVmcs <= X86_PAGE_4K_SIZE)
1909	{ /* likely */ }
1910	else
1911	{
1912	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1913	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1914	}
1915
1916	/*
1917	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1918	*/
1919	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1920	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1921	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1922	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1923	#endif
1924
1925	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1926	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1927	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1928
1929	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1930	{
1931	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1932	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1933	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1934	}
1935
1936	/*
1937	* Allocate per-VM VT-x structures.
1938	*/
1939	int rc = VINF_SUCCESS;
1940	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1941	/* Allocate crash-dump magic scratch page. */
1942	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1943	if (RT_FAILURE(rc))
1944	{
1945	hmR0VmxStructsFree(pVM);
1946	return rc;
1947	}
1948	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1949	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1950	#endif
1951
1952	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1953	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1954	{
1955	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1956	&pVM->hm.s.vmx.HCPhysApicAccess);
1957	if (RT_FAILURE(rc))
1958	{
1959	hmR0VmxStructsFree(pVM);
1960	return rc;
1961	}
1962	}
1963
1964	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1965	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1966	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1967	{
1968	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1969	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1970	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1971	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1972	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1973	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1974	{
1975	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1976	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1977	if (RT_SUCCESS(rc))
1978	{
1979	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1980	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1981	}
1982	}
1983	else
1984	rc = VERR_NO_MEMORY;
1985
1986	if (RT_FAILURE(rc))
1987	{
1988	hmR0VmxStructsFree(pVM);
1989	return rc;
1990	}
1991	}
1992	#endif
1993
1994	/*
1995	* Initialize per-VCPU VT-x structures.
1996	*/
1997	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1998	{
1999	/* Allocate the guest VMCS structures. */
2000	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2001	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2002	if (RT_SUCCESS(rc))
2003	{
2004	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2005	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2006	if (pVM->cpum.ro.GuestFeatures.fVmx)
2007	{
2008	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2009	if (RT_SUCCESS(rc))
2010	{ /* likely */ }
2011	else
2012	break;
2013	}
2014	#endif
2015	}
2016	else
2017	break;
2018	}
2019
2020	if (RT_FAILURE(rc))
2021	{
2022	hmR0VmxStructsFree(pVM);
2023	return rc;
2024	}
2025
2026	return VINF_SUCCESS;
2027	}
2028
2029	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2030	/**
2031	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2032	*
2033	* @returns @c true if the MSR is intercepted, @c false otherwise.
2034	* @param pvMsrBitmap The MSR bitmap.
2035	* @param offMsr The MSR byte offset.
2036	* @param iBit The bit offset from the byte offset.
2037	*/
2038	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2039	{
2040	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2041	Assert(pbMsrBitmap);
2042	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2043	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2044	}
2045	#endif
2046
2047	/**
2048	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2049	*
2050	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2051	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2052	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2053	* the read/write access of this MSR.
2054	*
2055	* @param pVCpu The cross context virtual CPU structure.
2056	* @param pVmcsInfo The VMCS info. object.
2057	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2058	* @param idMsr The MSR value.
2059	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2060	* include both a read -and- a write permission!
2061	*
2062	* @sa CPUMGetVmxMsrPermission.
2063	* @remarks Can be called with interrupts disabled.
2064	*/
2065	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2066	{
2067	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2068	Assert(pbMsrBitmap);
2069	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2070
2071	/*
2072	* MSR-bitmap Layout:
2073	* Byte index MSR range Interpreted as
2074	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2075	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2076	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2077	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2078	*
2079	* A bit corresponding to an MSR within the above range causes a VM-exit
2080	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2081	* the MSR range, it always cause a VM-exit.
2082	*
2083	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2084	*/
2085	uint16_t const offBitmapRead = 0;
2086	uint16_t const offBitmapWrite = 0x800;
2087	uint16_t offMsr;
2088	int32_t iBit;
2089	if (idMsr <= UINT32_C(0x00001fff))
2090	{
2091	offMsr = 0;
2092	iBit = idMsr;
2093	}
2094	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2095	{
2096	offMsr = 0x400;
2097	iBit = idMsr - UINT32_C(0xc0000000);
2098	}
2099	else
2100	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2101
2102	/*
2103	* Set the MSR read permission.
2104	*/
2105	uint16_t const offMsrRead = offBitmapRead + offMsr;
2106	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2107	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2108	{
2109	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2110	bool const fClear = !fIsNstGstVmcs ? true
2111	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2112	#else
2113	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2114	bool const fClear = true;
2115	#endif
2116	if (fClear)
2117	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2118	}
2119	else
2120	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2121
2122	/*
2123	* Set the MSR write permission.
2124	*/
2125	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2126	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2127	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2128	{
2129	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2130	bool const fClear = !fIsNstGstVmcs ? true
2131	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2132	#else
2133	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2134	bool const fClear = true;
2135	#endif
2136	if (fClear)
2137	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2138	}
2139	else
2140	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2141	}
2142
2143
2144	/**
2145	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2146	* area.
2147	*
2148	* @returns VBox status code.
2149	* @param pVCpu The cross context virtual CPU structure.
2150	* @param pVmcsInfo The VMCS info. object.
2151	* @param cMsrs The number of MSRs.
2152	*/
2153	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2154	{
2155	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2156	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2157	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2158	{
2159	/* Commit the MSR counts to the VMCS and update the cache. */
2160	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2161	{
2162	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2163	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2164	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2165	AssertRCReturn(rc, rc);
2166
2167	pVmcsInfo->cEntryMsrLoad = cMsrs;
2168	pVmcsInfo->cExitMsrStore = cMsrs;
2169	pVmcsInfo->cExitMsrLoad = cMsrs;
2170	}
2171	return VINF_SUCCESS;
2172	}
2173
2174	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2175	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2176	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2177	}
2178
2179
2180	/**
2181	* Adds a new (or updates the value of an existing) guest/host MSR
2182	* pair to be swapped during the world-switch as part of the
2183	* auto-load/store MSR area in the VMCS.
2184	*
2185	* @returns VBox status code.
2186	* @param pVCpu The cross context virtual CPU structure.
2187	* @param pVmxTransient The VMX-transient structure.
2188	* @param idMsr The MSR.
2189	* @param uGuestMsrValue Value of the guest MSR.
2190	* @param fSetReadWrite Whether to set the guest read/write access of this
2191	* MSR (thus not causing a VM-exit).
2192	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2193	* necessary.
2194	*/
2195	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2196	bool fSetReadWrite, bool fUpdateHostMsr)
2197	{
2198	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2199	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2200	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2201	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2202	uint32_t i;
2203
2204	/* Paranoia. */
2205	Assert(pGuestMsrLoad);
2206
2207	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2208
2209	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2210	for (i = 0; i < cMsrs; i++)
2211	{
2212	if (pGuestMsrLoad[i].u32Msr == idMsr)
2213	break;
2214	}
2215
2216	bool fAdded = false;
2217	if (i == cMsrs)
2218	{
2219	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2220	++cMsrs;
2221	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2222	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2223
2224	/* Set the guest to read/write this MSR without causing VM-exits. */
2225	if ( fSetReadWrite
2226	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2227	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2228
2229	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2230	fAdded = true;
2231	}
2232
2233	/* Update the MSR value for the newly added or already existing MSR. */
2234	pGuestMsrLoad[i].u32Msr = idMsr;
2235	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2236
2237	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2238	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2239	{
2240	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2241	pGuestMsrStore[i].u32Msr = idMsr;
2242	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2243	}
2244
2245	/* Update the corresponding slot in the host MSR area. */
2246	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2247	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2248	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2249	pHostMsr[i].u32Msr = idMsr;
2250
2251	/*
2252	* Only if the caller requests to update the host MSR value AND we've newly added the
2253	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2254	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2255	*
2256	* We do this for performance reasons since reading MSRs may be quite expensive.
2257	*/
2258	if (fAdded)
2259	{
2260	if (fUpdateHostMsr)
2261	{
2262	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2263	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2264	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2265	}
2266	else
2267	{
2268	/* Someone else can do the work. */
2269	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2270	}
2271	}
2272	return VINF_SUCCESS;
2273	}
2274
2275
2276	/**
2277	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2278	* auto-load/store MSR area in the VMCS.
2279	*
2280	* @returns VBox status code.
2281	* @param pVCpu The cross context virtual CPU structure.
2282	* @param pVmxTransient The VMX-transient structure.
2283	* @param idMsr The MSR.
2284	*/
2285	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2286	{
2287	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2288	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2289	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2290	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2291
2292	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2293
2294	for (uint32_t i = 0; i < cMsrs; i++)
2295	{
2296	/* Find the MSR. */
2297	if (pGuestMsrLoad[i].u32Msr == idMsr)
2298	{
2299	/*
2300	* If it's the last MSR, we only need to reduce the MSR count.
2301	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2302	*/
2303	if (i < cMsrs - 1)
2304	{
2305	/* Remove it from the VM-entry MSR-load area. */
2306	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2307	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2308
2309	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2310	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2311	{
2312	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2313	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2314	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2315	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2316	}
2317
2318	/* Remove it from the VM-exit MSR-load area. */
2319	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2320	Assert(pHostMsr[i].u32Msr == idMsr);
2321	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2322	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2323	}
2324
2325	/* Reduce the count to reflect the removed MSR and bail. */
2326	--cMsrs;
2327	break;
2328	}
2329	}
2330
2331	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2332	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2333	{
2334	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2335	AssertRCReturn(rc, rc);
2336
2337	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2338	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2339	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2340
2341	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2342	return VINF_SUCCESS;
2343	}
2344
2345	return VERR_NOT_FOUND;
2346	}
2347
2348
2349	/**
2350	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2351	*
2352	* @returns @c true if found, @c false otherwise.
2353	* @param pVmcsInfo The VMCS info. object.
2354	* @param idMsr The MSR to find.
2355	*/
2356	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2357	{
2358	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2359	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2360	Assert(pMsrs);
2361	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2362	for (uint32_t i = 0; i < cMsrs; i++)
2363	{
2364	if (pMsrs[i].u32Msr == idMsr)
2365	return true;
2366	}
2367	return false;
2368	}
2369
2370
2371	/**
2372	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2373	*
2374	* @param pVCpu The cross context virtual CPU structure.
2375	* @param pVmcsInfo The VMCS info. object.
2376	*
2377	* @remarks No-long-jump zone!!!
2378	*/
2379	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2380	{
2381	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2382
2383	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2384	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2385	Assert(pHostMsrLoad);
2386	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2387	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2388	for (uint32_t i = 0; i < cMsrs; i++)
2389	{
2390	/*
2391	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2392	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2393	*/
2394	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2395	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2396	else
2397	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2398	}
2399	}
2400
2401
2402	/**
2403	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2404	* perform lazy restoration of the host MSRs while leaving VT-x.
2405	*
2406	* @param pVCpu The cross context virtual CPU structure.
2407	*
2408	* @remarks No-long-jump zone!!!
2409	*/
2410	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2411	{
2412	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2413
2414	/*
2415	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2416	*/
2417	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2418	{
2419	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2420	#if HC_ARCH_BITS == 64
2421	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2422	{
2423	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2424	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2425	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2426	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2427	}
2428	#endif
2429	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2430	}
2431	}
2432
2433
2434	/**
2435	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2436	* lazily while leaving VT-x.
2437	*
2438	* @returns true if it does, false otherwise.
2439	* @param pVCpu The cross context virtual CPU structure.
2440	* @param idMsr The MSR to check.
2441	*/
2442	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2443	{
2444	NOREF(pVCpu);
2445	#if HC_ARCH_BITS == 64
2446	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2447	{
2448	switch (idMsr)
2449	{
2450	case MSR_K8_LSTAR:
2451	case MSR_K6_STAR:
2452	case MSR_K8_SF_MASK:
2453	case MSR_K8_KERNEL_GS_BASE:
2454	return true;
2455	}
2456	}
2457	#else
2458	RT_NOREF(pVCpu, idMsr);
2459	#endif
2460	return false;
2461	}
2462
2463
2464	/**
2465	* Loads a set of guests MSRs to allow read/passthru to the guest.
2466	*
2467	* The name of this function is slightly confusing. This function does NOT
2468	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2469	* common prefix for functions dealing with "lazy restoration" of the shared
2470	* MSRs.
2471	*
2472	* @param pVCpu The cross context virtual CPU structure.
2473	*
2474	* @remarks No-long-jump zone!!!
2475	*/
2476	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2477	{
2478	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2479	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2480
2481	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2482	#if HC_ARCH_BITS == 64
2483	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2484	{
2485	/*
2486	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2487	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2488	* we can skip a few MSR writes.
2489	*
2490	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2491	* guest MSR values in the guest-CPU context might be different to what's currently
2492	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2493	* CPU, see @bugref{8728}.
2494	*/
2495	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2496	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2497	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2498	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2499	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2500	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2501	{
2502	#ifdef VBOX_STRICT
2503	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2504	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2505	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2506	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2507	#endif
2508	}
2509	else
2510	{
2511	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2512	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2513	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2514	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2515	}
2516	}
2517	#endif
2518	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2519	}
2520
2521
2522	/**
2523	* Performs lazy restoration of the set of host MSRs if they were previously
2524	* loaded with guest MSR values.
2525	*
2526	* @param pVCpu The cross context virtual CPU structure.
2527	*
2528	* @remarks No-long-jump zone!!!
2529	* @remarks The guest MSRs should have been saved back into the guest-CPU
2530	* context by hmR0VmxImportGuestState()!!!
2531	*/
2532	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2533	{
2534	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2535	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2536
2537	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2538	{
2539	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2540	#if HC_ARCH_BITS == 64
2541	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2542	{
2543	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2544	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2545	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2546	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2547	}
2548	#endif
2549	}
2550	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2551	}
2552
2553
2554	/**
2555	* Verifies that our cached values of the VMCS fields are all consistent with
2556	* what's actually present in the VMCS.
2557	*
2558	* @returns VBox status code.
2559	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2560	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2561	* VMCS content. HMCPU error-field is
2562	* updated, see VMX_VCI_XXX.
2563	* @param pVCpu The cross context virtual CPU structure.
2564	* @param pVmcsInfo The VMCS info. object.
2565	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2566	*/
2567	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2568	{
2569	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2570
2571	uint32_t u32Val;
2572	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2573	AssertRCReturn(rc, rc);
2574	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2575	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2576	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2577	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2578
2579	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2580	AssertRCReturn(rc, rc);
2581	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2582	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2583	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2584	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2585
2586	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2587	AssertRCReturn(rc, rc);
2588	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2589	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2590	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2591	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2592
2593	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2594	AssertRCReturn(rc, rc);
2595	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2596	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2597	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2598	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2599
2600	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2601	{
2602	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2603	AssertRCReturn(rc, rc);
2604	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2605	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2606	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2607	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2608	}
2609
2610	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2611	AssertRCReturn(rc, rc);
2612	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2613	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2614	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2615	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2616
2617	uint64_t u64Val;
2618	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2619	AssertRCReturn(rc, rc);
2620	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2621	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2622	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2623	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2624
2625	NOREF(pcszVmcs);
2626	return VINF_SUCCESS;
2627	}
2628
2629
2630	#ifdef VBOX_STRICT
2631	/**
2632	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2633	*
2634	* @param pVCpu The cross context virtual CPU structure.
2635	* @param pVmcsInfo The VMCS info. object.
2636	*/
2637	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2638	{
2639	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2640
2641	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2642	{
2643	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2644	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2645	uint64_t uVmcsEferMsrVmcs;
2646	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2647	AssertRC(rc);
2648
2649	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2650	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2651	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2652	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2653	}
2654	}
2655
2656
2657	/**
2658	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2659	* VMCS are correct.
2660	*
2661	* @param pVCpu The cross context virtual CPU structure.
2662	* @param pVmcsInfo The VMCS info. object.
2663	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2664	*/
2665	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2666	{
2667	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2668
2669	/* Read the various MSR-area counts from the VMCS. */
2670	uint32_t cEntryLoadMsrs;
2671	uint32_t cExitStoreMsrs;
2672	uint32_t cExitLoadMsrs;
2673	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2676
2677	/* Verify all the MSR counts are the same. */
2678	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2679	Assert(cExitStoreMsrs == cExitLoadMsrs);
2680	uint32_t const cMsrs = cExitLoadMsrs;
2681
2682	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2683	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2684
2685	/* Verify the MSR counts are within the allocated page size. */
2686	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2687
2688	/* Verify the relevant contents of the MSR areas match. */
2689	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2690	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2691	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2692	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2693	for (uint32_t i = 0; i < cMsrs; i++)
2694	{
2695	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2696	if (fSeparateExitMsrStorePage)
2697	{
2698	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2699	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2700	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2701	}
2702
2703	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2704	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2705	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2706
2707	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2708	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2709	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2710	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2711
2712	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2713	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2714	if (fIsEferMsr)
2715	{
2716	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2717	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2718	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2719	}
2720
2721	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2722	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2723	{
2724	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2725	if (fIsEferMsr)
2726	{
2727	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2728	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2729	}
2730	else
2731	{
2732	if (!fIsNstGstVmcs)
2733	{
2734	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2735	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2736	}
2737	else
2738	{
2739	/*
2740	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2741	* execute a nested-guest with MSR passthrough.
2742	*
2743	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2744	* allow passthrough too.
2745	*/
2746	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2747	Assert(pvMsrBitmapNstGst);
2748	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2749	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2750	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2751	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2752	}
2753	}
2754	}
2755
2756	/* Move to the next MSR. */
2757	pHostMsrLoad++;
2758	pGuestMsrLoad++;
2759	pGuestMsrStore++;
2760	}
2761	}
2762	#endif /* VBOX_STRICT */
2763
2764
2765	/**
2766	* Flushes the TLB using EPT.
2767	*
2768	* @returns VBox status code.
2769	* @param pVCpu The cross context virtual CPU structure of the calling
2770	* EMT. Can be NULL depending on @a enmTlbFlush.
2771	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2772	* enmTlbFlush.
2773	* @param enmTlbFlush Type of flush.
2774	*
2775	* @remarks Caller is responsible for making sure this function is called only
2776	* when NestedPaging is supported and providing @a enmTlbFlush that is
2777	* supported by the CPU.
2778	* @remarks Can be called with interrupts disabled.
2779	*/
2780	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2781	{
2782	uint64_t au64Descriptor[2];
2783	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2784	au64Descriptor[0] = 0;
2785	else
2786	{
2787	Assert(pVCpu);
2788	Assert(pVmcsInfo);
2789	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2790	}
2791	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2792
2793	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2794	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2795
2796	if ( RT_SUCCESS(rc)
2797	&& pVCpu)
2798	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2799	}
2800
2801
2802	/**
2803	* Flushes the TLB using VPID.
2804	*
2805	* @returns VBox status code.
2806	* @param pVCpu The cross context virtual CPU structure of the calling
2807	* EMT. Can be NULL depending on @a enmTlbFlush.
2808	* @param enmTlbFlush Type of flush.
2809	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2810	* on @a enmTlbFlush).
2811	*
2812	* @remarks Can be called with interrupts disabled.
2813	*/
2814	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2815	{
2816	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2817
2818	uint64_t au64Descriptor[2];
2819	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2820	{
2821	au64Descriptor[0] = 0;
2822	au64Descriptor[1] = 0;
2823	}
2824	else
2825	{
2826	AssertPtr(pVCpu);
2827	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2828	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2829	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2830	au64Descriptor[1] = GCPtr;
2831	}
2832
2833	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2834	AssertMsg(rc == VINF_SUCCESS,
2835	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2836
2837	if ( RT_SUCCESS(rc)
2838	&& pVCpu)
2839	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2840	NOREF(rc);
2841	}
2842
2843
2844	/**
2845	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2846	* otherwise there is nothing really to invalidate.
2847	*
2848	* @returns VBox status code.
2849	* @param pVCpu The cross context virtual CPU structure.
2850	* @param GCVirt Guest virtual address of the page to invalidate.
2851	*/
2852	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2853	{
2854	AssertPtr(pVCpu);
2855	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2856
2857	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2858	{
2859	/*
2860	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2861	* the EPT case. See @bugref{6043} and @bugref{6177}.
2862	*
2863	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2864	* as this function maybe called in a loop with individual addresses.
2865	*/
2866	PVM pVM = pVCpu->CTX_SUFF(pVM);
2867	if (pVM->hm.s.vmx.fVpid)
2868	{
2869	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2870
2871	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2872	/*
2873	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2874	* where executing INVVPID outside 64-bit mode does not flush translations of
2875	* 64-bit linear addresses, see @bugref{6208#c72}.
2876	*/
2877	if (RT_HI_U32(GCVirt))
2878	fVpidFlush = false;
2879	#endif
2880
2881	if (fVpidFlush)
2882	{
2883	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2884	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2885	}
2886	else
2887	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2888	}
2889	else if (pVM->hm.s.fNestedPaging)
2890	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2891	}
2892
2893	return VINF_SUCCESS;
2894	}
2895
2896
2897	/**
2898	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2899	* case where neither EPT nor VPID is supported by the CPU.
2900	*
2901	* @param pHostCpu The HM physical-CPU structure.
2902	* @param pVCpu The cross context virtual CPU structure.
2903	*
2904	* @remarks Called with interrupts disabled.
2905	*/
2906	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2907	{
2908	AssertPtr(pVCpu);
2909	AssertPtr(pHostCpu);
2910
2911	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2912
2913	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2914	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2915	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2916	pVCpu->hm.s.fForceTLBFlush = false;
2917	return;
2918	}
2919
2920
2921	/**
2922	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2923	*
2924	* @param pHostCpu The HM physical-CPU structure.
2925	* @param pVCpu The cross context virtual CPU structure.
2926	* @param pVmcsInfo The VMCS info. object.
2927	*
2928	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2929	* nomenclature. The reason is, to avoid confusion in compare statements
2930	* since the host-CPU copies are named "ASID".
2931	*
2932	* @remarks Called with interrupts disabled.
2933	*/
2934	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2935	{
2936	#ifdef VBOX_WITH_STATISTICS
2937	bool fTlbFlushed = false;
2938	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2939	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2940	if (!fTlbFlushed) \
2941	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2942	} while (0)
2943	#else
2944	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2945	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2946	#endif
2947
2948	AssertPtr(pVCpu);
2949	AssertPtr(pHostCpu);
2950	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2951
2952	PVM pVM = pVCpu->CTX_SUFF(pVM);
2953	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2954	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2955	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2956
2957	/*
2958	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2959	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2960	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2961	* cannot reuse the current ASID anymore.
2962	*/
2963	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2964	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2965	{
2966	++pHostCpu->uCurrentAsid;
2967	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2968	{
2969	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2970	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2971	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2972	}
2973
2974	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2975	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2976	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2977
2978	/*
2979	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2980	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2981	*/
2982	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2983	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2984	HMVMX_SET_TAGGED_TLB_FLUSHED();
2985	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2986	}
2987	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2988	{
2989	/*
2990	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2991	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2992	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2993	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2994	* mappings, see @bugref{6568}.
2995	*
2996	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2997	*/
2998	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2999	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3000	HMVMX_SET_TAGGED_TLB_FLUSHED();
3001	}
3002
3003	pVCpu->hm.s.fForceTLBFlush = false;
3004	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3005
3006	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3007	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3008	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3009	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3010	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3011	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3012	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3013	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3014	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3015
3016	/* Update VMCS with the VPID. */
3017	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3018	AssertRC(rc);
3019
3020	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3021	}
3022
3023
3024	/**
3025	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3026	*
3027	* @param pHostCpu The HM physical-CPU structure.
3028	* @param pVCpu The cross context virtual CPU structure.
3029	* @param pVmcsInfo The VMCS info. object.
3030	*
3031	* @remarks Called with interrupts disabled.
3032	*/
3033	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3034	{
3035	AssertPtr(pVCpu);
3036	AssertPtr(pHostCpu);
3037	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3038	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3039	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3040
3041	/*
3042	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3043	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3044	*/
3045	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3046	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3047	{
3048	pVCpu->hm.s.fForceTLBFlush = true;
3049	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3050	}
3051
3052	/* Check for explicit TLB flushes. */
3053	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3054	{
3055	pVCpu->hm.s.fForceTLBFlush = true;
3056	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3057	}
3058
3059	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3060	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3061
3062	if (pVCpu->hm.s.fForceTLBFlush)
3063	{
3064	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3065	pVCpu->hm.s.fForceTLBFlush = false;
3066	}
3067	}
3068
3069
3070	/**
3071	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3072	*
3073	* @param pHostCpu The HM physical-CPU structure.
3074	* @param pVCpu The cross context virtual CPU structure.
3075	*
3076	* @remarks Called with interrupts disabled.
3077	*/
3078	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3079	{
3080	AssertPtr(pVCpu);
3081	AssertPtr(pHostCpu);
3082	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3083	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3084	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3085
3086	/*
3087	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3088	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3089	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3090	* cannot reuse the current ASID anymore.
3091	*/
3092	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3093	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3094	{
3095	pVCpu->hm.s.fForceTLBFlush = true;
3096	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3097	}
3098
3099	/* Check for explicit TLB flushes. */
3100	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3101	{
3102	/*
3103	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3104	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3105	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3106	* include fExplicitFlush's too) - an obscure corner case.
3107	*/
3108	pVCpu->hm.s.fForceTLBFlush = true;
3109	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3110	}
3111
3112	PVM pVM = pVCpu->CTX_SUFF(pVM);
3113	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3114	if (pVCpu->hm.s.fForceTLBFlush)
3115	{
3116	++pHostCpu->uCurrentAsid;
3117	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3118	{
3119	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3120	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3121	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3122	}
3123
3124	pVCpu->hm.s.fForceTLBFlush = false;
3125	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3126	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3127	if (pHostCpu->fFlushAsidBeforeUse)
3128	{
3129	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3130	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3131	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3132	{
3133	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3134	pHostCpu->fFlushAsidBeforeUse = false;
3135	}
3136	else
3137	{
3138	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3139	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3140	}
3141	}
3142	}
3143
3144	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3145	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3146	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3147	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3148	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3149	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3150	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3151
3152	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3153	AssertRC(rc);
3154	}
3155
3156
3157	/**
3158	* Flushes the guest TLB entry based on CPU capabilities.
3159	*
3160	* @param pHostCpu The HM physical-CPU structure.
3161	* @param pVCpu The cross context virtual CPU structure.
3162	* @param pVmcsInfo The VMCS info. object.
3163	*
3164	* @remarks Called with interrupts disabled.
3165	*/
3166	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3167	{
3168	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3169	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3170	#endif
3171	PVM pVM = pVCpu->CTX_SUFF(pVM);
3172	switch (pVM->hm.s.vmx.enmTlbFlushType)
3173	{
3174	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3175	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3176	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3177	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3178	default:
3179	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3180	break;
3181	}
3182	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3183	}
3184
3185
3186	/**
3187	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3188	* TLB entries from the host TLB before VM-entry.
3189	*
3190	* @returns VBox status code.
3191	* @param pVM The cross context VM structure.
3192	*/
3193	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3194	{
3195	/*
3196	* Determine optimal flush type for nested paging.
3197	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3198	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3199	*/
3200	if (pVM->hm.s.fNestedPaging)
3201	{
3202	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3203	{
3204	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3205	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3206	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3207	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3208	else
3209	{
3210	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3211	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3212	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3213	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3214	}
3215
3216	/* Make sure the write-back cacheable memory type for EPT is supported. */
3217	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3218	{
3219	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3220	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3221	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3222	}
3223
3224	/* EPT requires a page-walk length of 4. */
3225	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3226	{
3227	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3228	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3229	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3230	}
3231	}
3232	else
3233	{
3234	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3235	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3236	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3237	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3238	}
3239	}
3240
3241	/*
3242	* Determine optimal flush type for VPID.
3243	*/
3244	if (pVM->hm.s.vmx.fVpid)
3245	{
3246	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3247	{
3248	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3249	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3250	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3251	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3252	else
3253	{
3254	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3255	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3256	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3257	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3258	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3259	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3260	pVM->hm.s.vmx.fVpid = false;
3261	}
3262	}
3263	else
3264	{
3265	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3266	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3267	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3268	pVM->hm.s.vmx.fVpid = false;
3269	}
3270	}
3271
3272	/*
3273	* Setup the handler for flushing tagged-TLBs.
3274	*/
3275	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3276	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3277	else if (pVM->hm.s.fNestedPaging)
3278	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3279	else if (pVM->hm.s.vmx.fVpid)
3280	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3281	else
3282	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3283	return VINF_SUCCESS;
3284	}
3285
3286
3287	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3288	/**
3289	* Sets up the shadow VMCS fields arrays.
3290	*
3291	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3292	* executing the guest.
3293	*
3294	* @returns VBox status code.
3295	* @param pVM The cross context VM structure.
3296	*/
3297	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3298	{
3299	/*
3300	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3301	* when the host does not support it.
3302	*/
3303	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3304	if ( !fGstVmwriteAll
3305	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3306	{ /* likely. */ }
3307	else
3308	{
3309	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3310	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3311	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3312	}
3313
3314	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3315	uint32_t cRwFields = 0;
3316	uint32_t cRoFields = 0;
3317	for (uint32_t i = 0; i < cVmcsFields; i++)
3318	{
3319	VMXVMCSFIELD VmcsField;
3320	VmcsField.u = g_aVmcsFields[i];
3321
3322	/*
3323	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3324	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3325	* in the shadow VMCS fields array as they would be redundant.
3326	*
3327	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3328	* we must not include it in the shadow VMCS fields array. Guests attempting to
3329	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3330	* the required behavior.
3331	*/
3332	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3333	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3334	{
3335	/*
3336	* Read-only fields are placed in a separate array so that while syncing shadow
3337	* VMCS fields later (which is more performance critical) we can avoid branches.
3338	*
3339	* However, if the guest can write to all fields (including read-only fields),
3340	* we treat it a as read/write field. Otherwise, writing to these fields would
3341	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3342	*/
3343	if ( fGstVmwriteAll
3344	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3345	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3346	else
3347	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3348	}
3349	}
3350
3351	/* Update the counts. */
3352	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3353	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3354	return VINF_SUCCESS;
3355	}
3356
3357
3358	/**
3359	* Sets up the VMREAD and VMWRITE bitmaps.
3360	*
3361	* @param pVM The cross context VM structure.
3362	*/
3363	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3364	{
3365	/*
3366	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3367	*/
3368	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3369	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3370	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3371	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3372	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3373
3374	/*
3375	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3376	* VMREAD and VMWRITE bitmaps.
3377	*/
3378	{
3379	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3380	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3381	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3382	{
3383	uint32_t const uVmcsField = paShadowVmcsFields[i];
3384	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3385	Assert(uVmcsField >> 3 < cbBitmap);
3386	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3387	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3388	}
3389	}
3390
3391	/*
3392	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3393	* if the host supports VMWRITE to all supported VMCS fields.
3394	*/
3395	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3396	{
3397	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3398	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3399	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3400	{
3401	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3402	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3403	Assert(uVmcsField >> 3 < cbBitmap);
3404	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3405	}
3406	}
3407	}
3408	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3409
3410
3411	/**
3412	* Sets up the virtual-APIC page address for the VMCS.
3413	*
3414	* @returns VBox status code.
3415	* @param pVCpu The cross context virtual CPU structure.
3416	* @param pVmcsInfo The VMCS info. object.
3417	*/
3418	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3419	{
3420	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3421	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3422	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3423	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3424	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3425	}
3426
3427
3428	/**
3429	* Sets up the MSR-bitmap address for the VMCS.
3430	*
3431	* @returns VBox status code.
3432	* @param pVCpu The cross context virtual CPU structure.
3433	* @param pVmcsInfo The VMCS info. object.
3434	*/
3435	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3436	{
3437	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3438	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3439	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3440	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3441	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3442	}
3443
3444
3445	/**
3446	* Sets up the APIC-access page address for the VMCS.
3447	*
3448	* @returns VBox status code.
3449	* @param pVCpu The cross context virtual CPU structure.
3450	*/
3451	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3452	{
3453	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3454	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3455	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3456	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3457	}
3458
3459
3460	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3461	/**
3462	* Sets up the VMREAD bitmap address for the VMCS.
3463	*
3464	* @returns VBox status code.
3465	* @param pVCpu The cross context virtual CPU structure.
3466	*/
3467	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3468	{
3469	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3470	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3471	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3472	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3473	}
3474
3475
3476	/**
3477	* Sets up the VMWRITE bitmap address for the VMCS.
3478	*
3479	* @returns VBox status code.
3480	* @param pVCpu The cross context virtual CPU structure.
3481	*/
3482	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3483	{
3484	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3485	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3486	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3487	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3488	}
3489	#endif
3490
3491
3492	/**
3493	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3494	* in the VMCS.
3495	*
3496	* @returns VBox status code.
3497	* @param pVCpu The cross context virtual CPU structure.
3498	* @param pVmcsInfo The VMCS info. object.
3499	*/
3500	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3501	{
3502	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3503
3504	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3505	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3506	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3507
3508	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3509	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3510	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3511
3512	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3513	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3514	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3515
3516	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3517	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3518	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3519	AssertRCReturn(rc, rc);
3520	return VINF_SUCCESS;
3521	}
3522
3523
3524	/**
3525	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3526	*
3527	* @param pVCpu The cross context virtual CPU structure.
3528	* @param pVmcsInfo The VMCS info. object.
3529	*/
3530	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3531	{
3532	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3533
3534	/*
3535	* The guest can access the following MSRs (read, write) without causing
3536	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3537	*/
3538	PVM pVM = pVCpu->CTX_SUFF(pVM);
3539	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3540	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3541	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3542	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3543	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3544
3545	/*
3546	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3547	* associated with then. We never need to intercept access (writes need to be
3548	* executed without causing a VM-exit, reads will #GP fault anyway).
3549	*
3550	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3551	* read/write them. We swap the the guest/host MSR value using the
3552	* auto-load/store MSR area.
3553	*/
3554	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3555	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3556	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3557	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3558	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3559	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3560
3561	#if HC_ARCH_BITS == 64
3562	/*
3563	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3564	* required for 64-bit guests.
3565	*/
3566	if (pVM->hm.s.fAllow64BitGuests)
3567	{
3568	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3569	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3570	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3571	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3572	}
3573	#endif
3574
3575	/*
3576	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3577	*/
3578	#ifdef VBOX_STRICT
3579	Assert(pVmcsInfo->pvMsrBitmap);
3580	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3581	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3582	#endif
3583	}
3584
3585
3586	/**
3587	* Sets up pin-based VM-execution controls in the VMCS.
3588	*
3589	* @returns VBox status code.
3590	* @param pVCpu The cross context virtual CPU structure.
3591	* @param pVmcsInfo The VMCS info. object.
3592	*/
3593	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3594	{
3595	PVM pVM = pVCpu->CTX_SUFF(pVM);
3596	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3597	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3598
3599	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3600	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3601
3602	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3603	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3604
3605	/* Enable the VMX-preemption timer. */
3606	if (pVM->hm.s.vmx.fUsePreemptTimer)
3607	{
3608	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3609	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3610	}
3611
3612	#if 0
3613	/* Enable posted-interrupt processing. */
3614	if (pVM->hm.s.fPostedIntrs)
3615	{
3616	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3617	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3618	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3619	}
3620	#endif
3621
3622	if ((fVal & fZap) != fVal)
3623	{
3624	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3625	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3626	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3627	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3628	}
3629
3630	/* Commit it to the VMCS and update our cache. */
3631	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3632	AssertRCReturn(rc, rc);
3633	pVmcsInfo->u32PinCtls = fVal;
3634
3635	return VINF_SUCCESS;
3636	}
3637
3638
3639	/**
3640	* Sets up secondary processor-based VM-execution controls in the VMCS.
3641	*
3642	* @returns VBox status code.
3643	* @param pVCpu The cross context virtual CPU structure.
3644	* @param pVmcsInfo The VMCS info. object.
3645	*/
3646	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3647	{
3648	PVM pVM = pVCpu->CTX_SUFF(pVM);
3649	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3650	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3651
3652	/* WBINVD causes a VM-exit. */
3653	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3654	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3655
3656	/* Enable EPT (aka nested-paging). */
3657	if (pVM->hm.s.fNestedPaging)
3658	fVal \|= VMX_PROC_CTLS2_EPT;
3659
3660	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3661	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3662	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3663	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3664	fVal \|= VMX_PROC_CTLS2_INVPCID;
3665
3666	/* Enable VPID. */
3667	if (pVM->hm.s.vmx.fVpid)
3668	fVal \|= VMX_PROC_CTLS2_VPID;
3669
3670	/* Enable unrestricted guest execution. */
3671	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3672	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3673
3674	#if 0
3675	if (pVM->hm.s.fVirtApicRegs)
3676	{
3677	/* Enable APIC-register virtualization. */
3678	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3679	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3680
3681	/* Enable virtual-interrupt delivery. */
3682	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3683	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3684	}
3685	#endif
3686
3687	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3688	where the TPR shadow resides. */
3689	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3690	* done dynamically. */
3691	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3692	{
3693	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3694	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3695	AssertRCReturn(rc, rc);
3696	}
3697
3698	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3699	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3700	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3701	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3702	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3703
3704	/* Enable Pause-Loop exiting. */
3705	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3706	&& pVM->hm.s.vmx.cPleGapTicks
3707	&& pVM->hm.s.vmx.cPleWindowTicks)
3708	{
3709	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3710
3711	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3712	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3713	AssertRCReturn(rc, rc);
3714	}
3715
3716	if ((fVal & fZap) != fVal)
3717	{
3718	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3719	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3720	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3721	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3722	}
3723
3724	/* Commit it to the VMCS and update our cache. */
3725	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3726	AssertRCReturn(rc, rc);
3727	pVmcsInfo->u32ProcCtls2 = fVal;
3728
3729	return VINF_SUCCESS;
3730	}
3731
3732
3733	/**
3734	* Sets up processor-based VM-execution controls in the VMCS.
3735	*
3736	* @returns VBox status code.
3737	* @param pVCpu The cross context virtual CPU structure.
3738	* @param pVmcsInfo The VMCS info. object.
3739	*/
3740	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3741	{
3742	PVM pVM = pVCpu->CTX_SUFF(pVM);
3743
3744	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3745	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3746
3747	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3748	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3749	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3750	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3751	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3752	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3753	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3754
3755	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3756	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3757	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3758	{
3759	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3760	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3761	}
3762
3763	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3764	if (!pVM->hm.s.fNestedPaging)
3765	{
3766	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3767	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3768	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3769	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3770	}
3771
3772	/* Use TPR shadowing if supported by the CPU. */
3773	if ( PDMHasApic(pVM)
3774	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3775	{
3776	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3777	/* CR8 writes cause a VM-exit based on TPR threshold. */
3778	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3779	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3780	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3781	AssertRCReturn(rc, rc);
3782	}
3783	else
3784	{
3785	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3786	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3787	if (pVM->hm.s.fAllow64BitGuests)
3788	{
3789	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3790	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3791	}
3792	}
3793
3794	/* Use MSR-bitmaps if supported by the CPU. */
3795	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3796	{
3797	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3798	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3799	AssertRCReturn(rc, rc);
3800	}
3801
3802	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3803	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3804	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3805
3806	if ((fVal & fZap) != fVal)
3807	{
3808	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3809	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3810	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3811	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3812	}
3813
3814	/* Commit it to the VMCS and update our cache. */
3815	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3816	AssertRCReturn(rc, rc);
3817	pVmcsInfo->u32ProcCtls = fVal;
3818
3819	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3820	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3821	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3822
3823	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3824	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3825	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3826
3827	/* Sanity check, should not really happen. */
3828	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3829	{ /* likely */ }
3830	else
3831	{
3832	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3833	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3834	}
3835
3836	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3837	return VINF_SUCCESS;
3838	}
3839
3840
3841	/**
3842	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3843	* Processor-based VM-execution) control fields in the VMCS.
3844	*
3845	* @returns VBox status code.
3846	* @param pVCpu The cross context virtual CPU structure.
3847	* @param pVmcsInfo The VMCS info. object.
3848	*/
3849	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3850	{
3851	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3852	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3853	{
3854	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3855	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3856	if (RT_SUCCESS(rc))
3857	{ /* likely */ }
3858	else
3859	{
3860	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3861	return rc;
3862	}
3863	}
3864	#endif
3865
3866	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3867	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3868	if (RT_SUCCESS(rc))
3869	{
3870	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3871	if (RT_SUCCESS(rc))
3872	{
3873	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3874	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3875	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3876	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3877	if (RT_SUCCESS(rc))
3878	{
3879	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3880	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3881	return VINF_SUCCESS;
3882	}
3883	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3884	}
3885	else
3886	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3887	}
3888	else
3889	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3890	return rc;
3891	}
3892
3893
3894	/**
3895	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3896	*
3897	* We shall setup those exception intercepts that don't change during the
3898	* lifetime of the VM here. The rest are done dynamically while loading the
3899	* guest state.
3900	*
3901	* @returns VBox status code.
3902	* @param pVCpu The cross context virtual CPU structure.
3903	* @param pVmcsInfo The VMCS info. object.
3904	*/
3905	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3906	{
3907	/*
3908	* The following exceptions are always intercepted:
3909	*
3910	* #AC - To prevent the guest from hanging the CPU.
3911	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3912	* recursive #DBs can cause a CPU hang.
3913	* #PF - To sync our shadow page tables when nested-paging is not used.
3914	*/
3915	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3916	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3917	\| RT_BIT(X86_XCPT_DB)
3918	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3919
3920	/* Commit it to the VMCS. */
3921	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3922	AssertRCReturn(rc, rc);
3923
3924	/* Update our cache of the exception bitmap. */
3925	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3926	return VINF_SUCCESS;
3927	}
3928
3929
3930	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3931	/**
3932	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3933	*
3934	* @returns VBox status code.
3935	* @param pVCpu The cross context virtual CPU structure.
3936	* @param pVmcsInfo The VMCS info. object.
3937	*/
3938	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3939	{
3940	PVM pVM = pVCpu->CTX_SUFF(pVM);
3941	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3942	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3943	if (RT_SUCCESS(rc))
3944	{
3945	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3946	if (RT_SUCCESS(rc))
3947	{
3948	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3949	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3950	if (RT_SUCCESS(rc))
3951	{
3952	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3953	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3954	if (RT_SUCCESS(rc))
3955	return VINF_SUCCESS;
3956
3957	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3958	}
3959	else
3960	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3961	}
3962	else
3963	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3964	}
3965	else
3966	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3967
3968	return rc;
3969	}
3970	#endif
3971
3972
3973	/**
3974	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3975	* VMX.
3976	*
3977	* @returns VBox status code.
3978	* @param pVCpu The cross context virtual CPU structure.
3979	* @param pVmcsInfo The VMCS info. object.
3980	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3981	*/
3982	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3983	{
3984	Assert(pVmcsInfo->pvVmcs);
3985	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3986
3987	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3988	PVM pVM = pVCpu->CTX_SUFF(pVM);
3989	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3990	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3991
3992	LogFlowFunc(("\n"));
3993
3994	/*
3995	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3996	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3997	*/
3998	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3999	if (RT_SUCCESS(rc))
4000	{
4001	rc = hmR0VmxLoadVmcs(pVmcsInfo);
4002	if (RT_SUCCESS(rc))
4003	{
4004	if (!fIsNstGstVmcs)
4005	{
4006	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4007	if (RT_SUCCESS(rc))
4008	{
4009	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4010	if (RT_SUCCESS(rc))
4011	{
4012	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4013	if (RT_SUCCESS(rc))
4014	{
4015	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4016	if (RT_SUCCESS(rc))
4017	{
4018	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4019	/*
4020	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4021	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4022	* making it fit for use when VMCS shadowing is later enabled.
4023	*/
4024	if (pVmcsInfo->pvShadowVmcs)
4025	{
4026	VMXVMCSREVID VmcsRevId;
4027	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4028	VmcsRevId.n.fIsShadowVmcs = 1;
4029	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4030	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4031	if (RT_SUCCESS(rc))
4032	{ /* likely */ }
4033	else
4034	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4035	}
4036	#endif
4037	}
4038	else
4039	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4040	}
4041	else
4042	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4043	}
4044	else
4045	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4046	}
4047	else
4048	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4049	}
4050	else
4051	{
4052	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4053	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4054	if (RT_SUCCESS(rc))
4055	{ /* likely */ }
4056	else
4057	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4058	#else
4059	AssertFailed();
4060	#endif
4061	}
4062	}
4063	else
4064	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4065	}
4066	else
4067	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4068
4069	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4070	if (RT_SUCCESS(rc))
4071	{
4072	rc = hmR0VmxClearVmcs(pVmcsInfo);
4073	if (RT_SUCCESS(rc))
4074	{ /* likely */ }
4075	else
4076	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4077	}
4078
4079	/*
4080	* Update the last-error record both for failures and success, so we
4081	* can propagate the status code back to ring-3 for diagnostics.
4082	*/
4083	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4084	NOREF(pszVmcs);
4085	return rc;
4086	}
4087
4088
4089	/**
4090	* Does global VT-x initialization (called during module initialization).
4091	*
4092	* @returns VBox status code.
4093	*/
4094	VMMR0DECL(int) VMXR0GlobalInit(void)
4095	{
4096	#ifdef HMVMX_USE_FUNCTION_TABLE
4097	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4098	# ifdef VBOX_STRICT
4099	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4100	Assert(g_apfnVMExitHandlers[i]);
4101	# endif
4102	#endif
4103	return VINF_SUCCESS;
4104	}
4105
4106
4107	/**
4108	* Does global VT-x termination (called during module termination).
4109	*/
4110	VMMR0DECL(void) VMXR0GlobalTerm()
4111	{
4112	/* Nothing to do currently. */
4113	}
4114
4115
4116	/**
4117	* Sets up and activates VT-x on the current CPU.
4118	*
4119	* @returns VBox status code.
4120	* @param pHostCpu The HM physical-CPU structure.
4121	* @param pVM The cross context VM structure. Can be
4122	* NULL after a host resume operation.
4123	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4124	* fEnabledByHost is @c true).
4125	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4126	* @a fEnabledByHost is @c true).
4127	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4128	* enable VT-x on the host.
4129	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4130	*/
4131	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4132	PCSUPHWVIRTMSRS pHwvirtMsrs)
4133	{
4134	AssertPtr(pHostCpu);
4135	AssertPtr(pHwvirtMsrs);
4136	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4137
4138	/* Enable VT-x if it's not already enabled by the host. */
4139	if (!fEnabledByHost)
4140	{
4141	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4142	if (RT_FAILURE(rc))
4143	return rc;
4144	}
4145
4146	/*
4147	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4148	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4149	* invalidated when flushing by VPID.
4150	*/
4151	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4152	{
4153	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4154	pHostCpu->fFlushAsidBeforeUse = false;
4155	}
4156	else
4157	pHostCpu->fFlushAsidBeforeUse = true;
4158
4159	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4160	++pHostCpu->cTlbFlushes;
4161
4162	return VINF_SUCCESS;
4163	}
4164
4165
4166	/**
4167	* Deactivates VT-x on the current CPU.
4168	*
4169	* @returns VBox status code.
4170	* @param pvCpuPage Pointer to the VMXON region.
4171	* @param HCPhysCpuPage Physical address of the VMXON region.
4172	*
4173	* @remarks This function should never be called when SUPR0EnableVTx() or
4174	* similar was used to enable VT-x on the host.
4175	*/
4176	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4177	{
4178	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4179
4180	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4181	return hmR0VmxLeaveRootMode();
4182	}
4183
4184
4185	/**
4186	* Does per-VM VT-x initialization.
4187	*
4188	* @returns VBox status code.
4189	* @param pVM The cross context VM structure.
4190	*/
4191	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4192	{
4193	AssertPtr(pVM);
4194	LogFlowFunc(("pVM=%p\n", pVM));
4195
4196	int rc = hmR0VmxStructsAlloc(pVM);
4197	if (RT_FAILURE(rc))
4198	{
4199	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4200	return rc;
4201	}
4202
4203	return VINF_SUCCESS;
4204	}
4205
4206
4207	/**
4208	* Does per-VM VT-x termination.
4209	*
4210	* @returns VBox status code.
4211	* @param pVM The cross context VM structure.
4212	*/
4213	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4214	{
4215	AssertPtr(pVM);
4216	LogFlowFunc(("pVM=%p\n", pVM));
4217
4218	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4219	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4220	{
4221	Assert(pVM->hm.s.vmx.pvScratch);
4222	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4223	}
4224	#endif
4225	hmR0VmxStructsFree(pVM);
4226	return VINF_SUCCESS;
4227	}
4228
4229
4230	/**
4231	* Sets up the VM for execution using hardware-assisted VMX.
4232	* This function is only called once per-VM during initialization.
4233	*
4234	* @returns VBox status code.
4235	* @param pVM The cross context VM structure.
4236	*/
4237	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4238	{
4239	AssertPtr(pVM);
4240	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4241
4242	LogFlowFunc(("pVM=%p\n", pVM));
4243
4244	/*
4245	* At least verify if VMX is enabled, since we can't check if we're in
4246	* VMX root mode or not without causing a #GP.
4247	*/
4248	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4249	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4250	{ /* likely */ }
4251	else
4252	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4253
4254	/*
4255	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4256	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4257	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4258	*/
4259	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4261	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4262	{
4263	LogRelFunc(("Invalid real-on-v86 state.\n"));
4264	return VERR_INTERNAL_ERROR;
4265	}
4266
4267	/* Initialize these always, see hmR3InitFinalizeR0().*/
4268	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4269	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4270
4271	/* Setup the tagged-TLB flush handlers. */
4272	int rc = hmR0VmxSetupTaggedTlb(pVM);
4273	if (RT_FAILURE(rc))
4274	{
4275	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4276	return rc;
4277	}
4278
4279	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4280	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4281	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4282	{
4283	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4284	if (RT_SUCCESS(rc))
4285	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4286	else
4287	{
4288	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4289	return rc;
4290	}
4291	}
4292	#endif
4293
4294	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4295	{
4296	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4297	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4298
4299	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4300	if (RT_SUCCESS(rc))
4301	{
4302	#if HC_ARCH_BITS == 32
4303	hmR0VmxInitVmcsReadCache(pVCpu);
4304	#endif
4305	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4306	if (pVM->cpum.ro.GuestFeatures.fVmx)
4307	{
4308	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4309	if (RT_SUCCESS(rc))
4310	{ /* likely */ }
4311	else
4312	{
4313	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4314	return rc;
4315	}
4316	}
4317	#endif
4318	}
4319	else
4320	{
4321	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4322	return rc;
4323	}
4324	}
4325
4326	return VINF_SUCCESS;
4327	}
4328
4329
4330	#if HC_ARCH_BITS == 32
4331	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4332	/**
4333	* Check if guest state allows safe use of 32-bit switcher again.
4334	*
4335	* Segment bases and protected mode structures must be 32-bit addressable
4336	* because the 32-bit switcher will ignore high dword when writing these VMCS
4337	* fields. See @bugref{8432} for details.
4338	*
4339	* @returns true if safe, false if must continue to use the 64-bit switcher.
4340	* @param pCtx Pointer to the guest-CPU context.
4341	*
4342	* @remarks No-long-jump zone!!!
4343	*/
4344	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4345	{
4346	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4347	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4348	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4349	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4350	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4351	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4352	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4353	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4354	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4355	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4356
4357	/* All good, bases are 32-bit. */
4358	return true;
4359	}
4360	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4361
4362	# ifdef VBOX_STRICT
4363	static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4364	{
4365	switch (idxField)
4366	{
4367	case VMX_VMCS_GUEST_RIP:
4368	case VMX_VMCS_GUEST_RSP:
4369	case VMX_VMCS_GUEST_SYSENTER_EIP:
4370	case VMX_VMCS_GUEST_SYSENTER_ESP:
4371	case VMX_VMCS_GUEST_GDTR_BASE:
4372	case VMX_VMCS_GUEST_IDTR_BASE:
4373	case VMX_VMCS_GUEST_CS_BASE:
4374	case VMX_VMCS_GUEST_DS_BASE:
4375	case VMX_VMCS_GUEST_ES_BASE:
4376	case VMX_VMCS_GUEST_FS_BASE:
4377	case VMX_VMCS_GUEST_GS_BASE:
4378	case VMX_VMCS_GUEST_SS_BASE:
4379	case VMX_VMCS_GUEST_LDTR_BASE:
4380	case VMX_VMCS_GUEST_TR_BASE:
4381	case VMX_VMCS_GUEST_CR3:
4382	return true;
4383	}
4384	return false;
4385	}
4386
4387	static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4388	{
4389	switch (idxField)
4390	{
4391	/* Read-only fields. */
4392	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4393	return true;
4394	}
4395	/* Remaining readable fields should also be writable. */
4396	return hmR0VmxIsValidWriteFieldInCache(idxField);
4397	}
4398	# endif /* VBOX_STRICT */
4399
4400
4401	/**
4402	* Executes the specified handler in 64-bit mode.
4403	*
4404	* @returns VBox status code (no informational status codes).
4405	* @param pVCpu The cross context virtual CPU structure.
4406	* @param enmOp The operation to perform.
4407	* @param cParams Number of parameters.
4408	* @param paParam Array of 32-bit parameters.
4409	*/
4410	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4411	{
4412	AssertPtr(pVCpu);
4413	PVM pVM = pVCpu->CTX_SUFF(pVM);
4414	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4415	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4416	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4417	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4418
4419	#ifdef VBOX_STRICT
4420	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4421	Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4422
4423	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4424	Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4425	#endif
4426
4427	/* Disable interrupts. */
4428	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4429
4430	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4431	RTCPUID idHostCpu = RTMpCpuId();
4432	CPUMR0SetLApic(pVCpu, idHostCpu);
4433	#endif
4434
4435	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4436
4437	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4438	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4439
4440	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4441	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4442	hmR0VmxClearVmcs(pVmcsInfo);
4443
4444	/* Leave VMX root mode and disable VMX. */
4445	VMXDisable();
4446	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4447
4448	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4449	CPUMSetHyperEIP(pVCpu, enmOp);
4450	for (int i = (int)cParams - 1; i >= 0; i--)
4451	CPUMPushHyper(pVCpu, paParam[i]);
4452
4453	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4454
4455	/* Call the switcher. */
4456	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4458
4459	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4460	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4461
4462	/* Re-enter VMX root mode. */
4463	int rc2 = VMXEnable(HCPhysCpuPage);
4464	if (RT_FAILURE(rc2))
4465	{
4466	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4467	ASMSetFlags(fOldEFlags);
4468	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4469	return rc2;
4470	}
4471
4472	/* Restore the VMCS as the current VMCS. */
4473	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4474	AssertRC(rc2);
4475	Assert(!(ASMGetFlags() & X86_EFL_IF));
4476	ASMSetFlags(fOldEFlags);
4477	return rc;
4478	}
4479
4480
4481	/**
4482	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4483	* supporting 64-bit guests.
4484	*
4485	* @returns VBox status code.
4486	* @param fResume Whether to VMLAUNCH or VMRESUME.
4487	* @param pCtx Pointer to the guest-CPU context.
4488	* @param pCache Pointer to the VMCS batch cache.
4489	* @param pVM The cross context VM structure.
4490	* @param pVCpu The cross context virtual CPU structure.
4491	*/
4492	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4493	{
4494	NOREF(fResume);
4495
4496	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4497	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4498	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4499
4500	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4501	pCache->uPos = 1;
4502	pCache->interPD = PGMGetInterPaeCR3(pVM);
4503	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4504	#endif
4505
4506	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4507	pCache->TestIn.HCPhysCpuPage = 0;
4508	pCache->TestIn.HCPhysVmcs = 0;
4509	pCache->TestIn.pCache = 0;
4510	pCache->TestOut.HCPhysVmcs = 0;
4511	pCache->TestOut.pCache = 0;
4512	pCache->TestOut.pCtx = 0;
4513	pCache->TestOut.eflags = 0;
4514	#else
4515	NOREF(pCache);
4516	#endif
4517
4518	uint32_t aParam[10];
4519	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4520	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4521	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4522	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4523	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4524	aParam[5] = 0;
4525	aParam[6] = VM_RC_ADDR(pVM, pVM);
4526	aParam[7] = 0;
4527	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4528	aParam[9] = 0;
4529
4530	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4531	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4532	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4533	#endif
4534	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4535
4536	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4537	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4538	Assert(pCtx->dr[4] == 10);
4539	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4540	#endif
4541
4542	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4543	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4544	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4545	pVmcsInfo->HCPhysVmcs));
4546	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4547	pCache->TestOut.HCPhysVmcs));
4548	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4549	pCache->TestOut.pCache));
4550	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4551	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4552	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4553	pCache->TestOut.pCtx));
4554	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4555	#endif
4556	NOREF(pCtx);
4557	return rc;
4558	}
4559	#endif
4560
4561
4562	/**
4563	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4564	* the VMCS.
4565	*
4566	* @returns VBox status code.
4567	*/
4568	static int hmR0VmxExportHostControlRegs(void)
4569	{
4570	RTCCUINTREG uReg = ASMGetCR0();
4571	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4572	AssertRCReturn(rc, rc);
4573
4574	uReg = ASMGetCR3();
4575	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4576	AssertRCReturn(rc, rc);
4577
4578	uReg = ASMGetCR4();
4579	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4580	AssertRCReturn(rc, rc);
4581	return rc;
4582	}
4583
4584
4585	/**
4586	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4587	* the host-state area in the VMCS.
4588	*
4589	* @returns VBox status code.
4590	* @param pVCpu The cross context virtual CPU structure.
4591	*/
4592	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4593	{
4594	#if HC_ARCH_BITS == 64
4595	/**
4596	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4597	* requirements. See hmR0VmxExportHostSegmentRegs().
4598	*/
4599	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4600	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4601	{ \
4602	bool fValidSelector = true; \
4603	if ((selValue) & X86_SEL_LDT) \
4604	{ \
4605	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4606	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4607	} \
4608	if (fValidSelector) \
4609	{ \
4610	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4611	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4612	} \
4613	(selValue) = 0; \
4614	}
4615
4616	/*
4617	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4618	* will be messed up. We should -not- save the messed up state without restoring
4619	* the original host-state, see @bugref{7240}.
4620	*
4621	* This apparently can happen (most likely the FPU changes), deal with it rather than
4622	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4623	*/
4624	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4625	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4626	{
4627	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4628	pVCpu->idCpu));
4629	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4630	}
4631	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4632	#else
4633	RT_NOREF(pVCpu);
4634	#endif
4635
4636	/*
4637	* Host DS, ES, FS and GS segment registers.
4638	*/
4639	#if HC_ARCH_BITS == 64
4640	RTSEL uSelDS = ASMGetDS();
4641	RTSEL uSelES = ASMGetES();
4642	RTSEL uSelFS = ASMGetFS();
4643	RTSEL uSelGS = ASMGetGS();
4644	#else
4645	RTSEL uSelDS = 0;
4646	RTSEL uSelES = 0;
4647	RTSEL uSelFS = 0;
4648	RTSEL uSelGS = 0;
4649	#endif
4650
4651	/*
4652	* Host CS and SS segment registers.
4653	*/
4654	RTSEL uSelCS = ASMGetCS();
4655	RTSEL uSelSS = ASMGetSS();
4656
4657	/*
4658	* Host TR segment register.
4659	*/
4660	RTSEL uSelTR = ASMGetTR();
4661
4662	#if HC_ARCH_BITS == 64
4663	/*
4664	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4665	* gain VM-entry and restore them before we get preempted.
4666	*
4667	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4668	*/
4669	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4670	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4671	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4672	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4673	# undef VMXLOCAL_ADJUST_HOST_SEG
4674	#endif
4675
4676	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4677	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4678	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4679	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4680	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4681	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4682	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4683	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4684	Assert(uSelCS);
4685	Assert(uSelTR);
4686
4687	/* Write these host selector fields into the host-state area in the VMCS. */
4688	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4689	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4690	#if HC_ARCH_BITS == 64
4691	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4692	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4693	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4694	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4695	#else
4696	NOREF(uSelDS);
4697	NOREF(uSelES);
4698	NOREF(uSelFS);
4699	NOREF(uSelGS);
4700	#endif
4701	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4702	AssertRCReturn(rc, rc);
4703
4704	/*
4705	* Host GDTR and IDTR.
4706	*/
4707	RTGDTR Gdtr;
4708	RTIDTR Idtr;
4709	RT_ZERO(Gdtr);
4710	RT_ZERO(Idtr);
4711	ASMGetGDTR(&Gdtr);
4712	ASMGetIDTR(&Idtr);
4713	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4714	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4715	AssertRCReturn(rc, rc);
4716
4717	#if HC_ARCH_BITS == 64
4718	/*
4719	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4720	* them to the maximum limit (0xffff) on every VM-exit.
4721	*/
4722	if (Gdtr.cbGdt != 0xffff)
4723	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4724
4725	/*
4726	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4727	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4728	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4729	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4730	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4731	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4732	* at 0xffff on hosts where we are sure it won't cause trouble.
4733	*/
4734	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4735	if (Idtr.cbIdt < 0x0fff)
4736	# else
4737	if (Idtr.cbIdt != 0xffff)
4738	# endif
4739	{
4740	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4741	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4742	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4743	}
4744	#endif
4745
4746	/*
4747	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4748	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4749	* RPL should be too in most cases.
4750	*/
4751	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4752	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4753
4754	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4755	#if HC_ARCH_BITS == 64
4756	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4757
4758	/*
4759	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4760	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4761	* restoration if the host has something else. Task switching is not supported in 64-bit
4762	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4763	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4764	*
4765	* [1] See Intel spec. 3.5 "System Descriptor Types".
4766	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4767	*/
4768	PVM pVM = pVCpu->CTX_SUFF(pVM);
4769	Assert(pDesc->System.u4Type == 11);
4770	if ( pDesc->System.u16LimitLow != 0x67
4771	\|\| pDesc->System.u4LimitHigh)
4772	{
4773	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4774	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4775	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4776	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4777	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4778	}
4779
4780	/*
4781	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4782	*/
4783	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4784	{
4785	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4786	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4787	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4788	{
4789	/* The GDT is read-only but the writable GDT is available. */
4790	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4791	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4792	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4793	AssertRCReturn(rc, rc);
4794	}
4795	}
4796	#else
4797	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4798	#endif
4799	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4800	AssertRCReturn(rc, rc);
4801
4802	/*
4803	* Host FS base and GS base.
4804	*/
4805	#if HC_ARCH_BITS == 64
4806	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4807	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4808	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4809	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4810	AssertRCReturn(rc, rc);
4811
4812	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4813	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4814	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4815	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4816	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4817	#endif
4818	return VINF_SUCCESS;
4819	}
4820
4821
4822	/**
4823	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4824	* host-state area of the VMCS.
4825	*
4826	* These MSRs will be automatically restored on the host after every successful
4827	* VM-exit.
4828	*
4829	* @returns VBox status code.
4830	* @param pVCpu The cross context virtual CPU structure.
4831	*
4832	* @remarks No-long-jump zone!!!
4833	*/
4834	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4835	{
4836	AssertPtr(pVCpu);
4837
4838	/*
4839	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4840	* rather than swapping them on every VM-entry.
4841	*/
4842	hmR0VmxLazySaveHostMsrs(pVCpu);
4843
4844	/*
4845	* Host Sysenter MSRs.
4846	*/
4847	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4848	#if HC_ARCH_BITS == 32
4849	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4850	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4851	#else
4852	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4853	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4854	#endif
4855	AssertRCReturn(rc, rc);
4856
4857	/*
4858	* Host EFER MSR.
4859	*
4860	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4861	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4862	*/
4863	PVM pVM = pVCpu->CTX_SUFF(pVM);
4864	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4865	{
4866	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4867	AssertRCReturn(rc, rc);
4868	}
4869
4870	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4871	* hmR0VmxExportGuestEntryExitCtls(). */
4872
4873	return VINF_SUCCESS;
4874	}
4875
4876
4877	/**
4878	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4879	*
4880	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4881	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4882	*
4883	* @returns true if we need to load guest EFER, false otherwise.
4884	* @param pVCpu The cross context virtual CPU structure.
4885	*
4886	* @remarks Requires EFER, CR4.
4887	* @remarks No-long-jump zone!!!
4888	*/
4889	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4890	{
4891	#ifdef HMVMX_ALWAYS_SWAP_EFER
4892	RT_NOREF(pVCpu);
4893	return true;
4894	#else
4895	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4896	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4897	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4898	if (CPUMIsGuestInLongModeEx(pCtx))
4899	return false;
4900	#endif
4901
4902	PVM pVM = pVCpu->CTX_SUFF(pVM);
4903	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4904	uint64_t const u64GuestEfer = pCtx->msrEFER;
4905
4906	/*
4907	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4908	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4909	*/
4910	if ( CPUMIsGuestInLongModeEx(pCtx)
4911	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4912	return true;
4913
4914	/*
4915	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4916	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4917	*
4918	* See Intel spec. 4.5 "IA-32e Paging".
4919	* See Intel spec. 4.1.1 "Three Paging Modes".
4920	*
4921	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4922	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4923	*/
4924	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4925	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4926	if ( (pCtx->cr4 & X86_CR4_PAE)
4927	&& (pCtx->cr0 & X86_CR0_PG)
4928	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4929	{
4930	/* Assert that host is NX capable. */
4931	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4932	return true;
4933	}
4934
4935	return false;
4936	#endif
4937	}
4938
4939	/**
4940	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4941	* VMCS.
4942	*
4943	* This is typically required when the guest changes paging mode.
4944	*
4945	* @returns VBox status code.
4946	* @param pVCpu The cross context virtual CPU structure.
4947	* @param pVmxTransient The VMX-transient structure.
4948	*
4949	* @remarks Requires EFER.
4950	* @remarks No-long-jump zone!!!
4951	*/
4952	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4953	{
4954	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4955	{
4956	PVM pVM = pVCpu->CTX_SUFF(pVM);
4957	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4958
4959	/*
4960	* VM-entry controls.
4961	*/
4962	{
4963	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4964	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4965
4966	/*
4967	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4968	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4969	*
4970	* For nested-guests, this is a mandatory VM-entry control. It's also
4971	* required because we do not want to leak host bits to the nested-guest.
4972	*/
4973	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4974
4975	/*
4976	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4977	*
4978	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4979	* required to get the nested-guest working with hardware-assisted VMX execution.
4980	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4981	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4982	* here rather than while merging the guest VMCS controls.
4983	*/
4984	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4985	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4986	else
4987	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4988
4989	/*
4990	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4991	*
4992	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4993	* regardless of whether the nested-guest VMCS specifies it because we are free to
4994	* load whatever MSRs we require and we do not need to modify the guest visible copy
4995	* of the VM-entry MSR load area.
4996	*/
4997	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4998	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4999	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
5000	else
5001	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
5002
5003	/*
5004	* The following should -not- be set (since we're not in SMM mode):
5005	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
5006	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
5007	*/
5008
5009	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
5010	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
5011
5012	if ((fVal & fZap) == fVal)
5013	{ /* likely */ }
5014	else
5015	{
5016	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
5017	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
5018	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
5019	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5020	}
5021
5022	/* Commit it to the VMCS. */
5023	if (pVmcsInfo->u32EntryCtls != fVal)
5024	{
5025	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
5026	AssertRCReturn(rc, rc);
5027	pVmcsInfo->u32EntryCtls = fVal;
5028	}
5029	}
5030
5031	/*
5032	* VM-exit controls.
5033	*/
5034	{
5035	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
5036	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
5037
5038	/*
5039	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
5040	* supported the 1-setting of this bit.
5041	*
5042	* For nested-guests, we set the "save debug controls" as the converse
5043	* "load debug controls" is mandatory for nested-guests anyway.
5044	*/
5045	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
5046
5047	/*
5048	* Set the host long mode active (EFER.LMA) bit (which Intel calls
5049	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
5050	* host EFER.LMA and EFER.LME bit to this value. See assertion in
5051	* hmR0VmxExportHostMsrs().
5052	*
5053	* For nested-guests, we always set this bit as we do not support 32-bit
5054	* hosts.
5055	*/
5056	#if HC_ARCH_BITS == 64
5057	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5058	#else
5059	Assert(!pVmxTransient->fIsNestedGuest);
5060	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5061	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5062	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5063	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5064	{
5065	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5066	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5067	}
5068	else
5069	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5070	#endif
5071
5072	/*
5073	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
5074	*
5075	* For nested-guests, we should use the "save IA32_EFER" control if we also
5076	* used the "load IA32_EFER" control while exporting VM-entry controls.
5077	*/
5078	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5079	&& hmR0VmxShouldSwapEferMsr(pVCpu))
5080	{
5081	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
5082	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
5083	}
5084
5085	/*
5086	* Enable saving of the VMX-preemption timer value on VM-exit.
5087	* For nested-guests, currently not exposed/used.
5088	*/
5089	if ( pVM->hm.s.vmx.fUsePreemptTimer
5090	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5091	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5092
5093	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5094	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5095
5096	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5097	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
5098	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5099
5100	if ((fVal & fZap) == fVal)
5101	{ /* likely */ }
5102	else
5103	{
5104	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5105	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5106	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5107	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5108	}
5109
5110	/* Commit it to the VMCS. */
5111	if (pVmcsInfo->u32ExitCtls != fVal)
5112	{
5113	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5114	AssertRCReturn(rc, rc);
5115	pVmcsInfo->u32ExitCtls = fVal;
5116	}
5117	}
5118
5119	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5120	}
5121	return VINF_SUCCESS;
5122	}
5123
5124
5125	/**
5126	* Sets the TPR threshold in the VMCS.
5127	*
5128	* @returns VBox status code.
5129	* @param pVCpu The cross context virtual CPU structure.
5130	* @param pVmcsInfo The VMCS info. object.
5131	* @param u32TprThreshold The TPR threshold (task-priority class only).
5132	*/
5133	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5134	{
5135	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5136	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5137	RT_NOREF2(pVCpu, pVmcsInfo);
5138	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5139	}
5140
5141
5142	/**
5143	* Exports the guest APIC TPR state into the VMCS.
5144	*
5145	* @returns VBox status code.
5146	* @param pVCpu The cross context virtual CPU structure.
5147	* @param pVmxTransient The VMX-transient structure.
5148	*
5149	* @remarks No-long-jump zone!!!
5150	*/
5151	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5152	{
5153	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5154	{
5155	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5156
5157	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5158	if (!pVmxTransient->fIsNestedGuest)
5159	{
5160	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5161	&& APICIsEnabled(pVCpu))
5162	{
5163	/*
5164	* Setup TPR shadowing.
5165	*/
5166	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5167	{
5168	bool fPendingIntr = false;
5169	uint8_t u8Tpr = 0;
5170	uint8_t u8PendingIntr = 0;
5171	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5172	AssertRCReturn(rc, rc);
5173
5174	/*
5175	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5176	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5177	* priority of the pending interrupt so we can deliver the interrupt. If there
5178	* are no interrupts pending, set threshold to 0 to not cause any
5179	* TPR-below-threshold VM-exits.
5180	*/
5181	Assert(pVmcsInfo->pbVirtApic);
5182	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5183	uint32_t u32TprThreshold = 0;
5184	if (fPendingIntr)
5185	{
5186	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5187	(which is the Task-Priority Class). */
5188	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5189	const uint8_t u8TprPriority = u8Tpr >> 4;
5190	if (u8PendingPriority <= u8TprPriority)
5191	u32TprThreshold = u8PendingPriority;
5192	}
5193
5194	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5195	AssertRCReturn(rc, rc);
5196	}
5197	}
5198	}
5199	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5200	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5201	}
5202	return VINF_SUCCESS;
5203	}
5204
5205
5206	/**
5207	* Gets the guest interruptibility-state.
5208	*
5209	* @returns Guest's interruptibility-state.
5210	* @param pVCpu The cross context virtual CPU structure.
5211	* @param pVmxTransient The VMX-transient structure.
5212	*
5213	* @remarks No-long-jump zone!!!
5214	*/
5215	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5216	{
5217	/*
5218	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5219	*/
5220	uint32_t fIntrState = 0;
5221	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5222	{
5223	/* If inhibition is active, RIP and RFLAGS should've been updated
5224	(i.e. read previously from the VMCS or from ring-3). */
5225	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5226	#ifdef VBOX_STRICT
5227	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5228	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5229	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5230	#endif
5231	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5232	{
5233	if (pCtx->eflags.Bits.u1IF)
5234	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5235	else
5236	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5237	}
5238	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5239	{
5240	/*
5241	* We can clear the inhibit force flag as even if we go back to the recompiler
5242	* without executing guest code in VT-x, the flag's condition to be cleared is
5243	* met and thus the cleared state is correct.
5244	*/
5245	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5246	}
5247	}
5248
5249	/*
5250	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5251	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5252	* setting this would block host-NMIs and IRET will not clear the blocking.
5253	*
5254	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5255	*
5256	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5257	*/
5258	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5259	&& CPUMIsGuestNmiBlocking(pVCpu))
5260	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5261
5262	return fIntrState;
5263	}
5264
5265
5266	/**
5267	* Exports the exception intercepts required for guest execution in the VMCS.
5268	*
5269	* @returns VBox status code.
5270	* @param pVCpu The cross context virtual CPU structure.
5271	* @param pVmxTransient The VMX-transient structure.
5272	*
5273	* @remarks No-long-jump zone!!!
5274	*/
5275	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5276	{
5277	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5278	{
5279	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5280	if ( !pVmxTransient->fIsNestedGuest
5281	&& pVCpu->hm.s.fGIMTrapXcptUD)
5282	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5283	else
5284	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5285
5286	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5287	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5288	}
5289	return VINF_SUCCESS;
5290	}
5291
5292
5293	/**
5294	* Exports the guest's RIP into the guest-state area in the VMCS.
5295	*
5296	* @returns VBox status code.
5297	* @param pVCpu The cross context virtual CPU structure.
5298	*
5299	* @remarks No-long-jump zone!!!
5300	*/
5301	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5302	{
5303	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5304	{
5305	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5306
5307	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5308	AssertRCReturn(rc, rc);
5309
5310	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5311	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5312	}
5313	return VINF_SUCCESS;
5314	}
5315
5316
5317	/**
5318	* Exports the guest's RSP into the guest-state area in the VMCS.
5319	*
5320	* @returns VBox status code.
5321	* @param pVCpu The cross context virtual CPU structure.
5322	*
5323	* @remarks No-long-jump zone!!!
5324	*/
5325	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5326	{
5327	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5328	{
5329	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5330
5331	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5332	AssertRCReturn(rc, rc);
5333
5334	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5335	}
5336	return VINF_SUCCESS;
5337	}
5338
5339
5340	/**
5341	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5342	*
5343	* @returns VBox status code.
5344	* @param pVCpu The cross context virtual CPU structure.
5345	* @param pVmxTransient The VMX-transient structure.
5346	*
5347	* @remarks No-long-jump zone!!!
5348	*/
5349	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5350	{
5351	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5352	{
5353	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5354
5355	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5356	Let us assert it as such and use 32-bit VMWRITE. */
5357	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5358	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5359	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5360	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5361
5362	/*
5363	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5364	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5365	* can run the real-mode guest code under Virtual 8086 mode.
5366	*/
5367	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5368	if (pVmcsInfo->RealMode.fRealOnV86Active)
5369	{
5370	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5371	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5372	Assert(!pVmxTransient->fIsNestedGuest);
5373	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5374	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5375	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5376	}
5377
5378	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5379	AssertRCReturn(rc, rc);
5380
5381	/*
5382	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5383	*
5384	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5385	* through the hypervisor debugger using EFLAGS.TF.
5386	*/
5387	if ( !pVmxTransient->fIsNestedGuest
5388	&& !pVCpu->hm.s.fSingleInstruction
5389	&& fEFlags.Bits.u1TF)
5390	{
5391	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5392	* premature trips to ring-3 esp since IEM does not yet handle it. */
5393	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5394	AssertRCReturn(rc, rc);
5395	}
5396	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5397	* nested-guest VMCS. */
5398
5399	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5400	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5401	}
5402	return VINF_SUCCESS;
5403	}
5404
5405
5406	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5407	/**
5408	* Copies the nested-guest VMCS to the shadow VMCS.
5409	*
5410	* @returns VBox status code.
5411	* @param pVCpu The cross context virtual CPU structure.
5412	* @param pVmcsInfo The VMCS info. object.
5413	*
5414	* @remarks No-long-jump zone!!!
5415	*/
5416	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5417	{
5418	PVM pVM = pVCpu->CTX_SUFF(pVM);
5419	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5420
5421	/*
5422	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5423	* current VMCS, as we may try saving guest lazy MSRs.
5424	*
5425	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5426	* calling the import VMCS code which is currently performing the guest MSR reads
5427	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5428	* and the rest of the VMX leave session machinery.
5429	*/
5430	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5431
5432	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5433	if (RT_SUCCESS(rc))
5434	{
5435	/*
5436	* Copy all guest read/write VMCS fields.
5437	*
5438	* We don't check for VMWRITE failures here for performance reasons and
5439	* because they are not expected to fail, barring irrecoverable conditions
5440	* like hardware errors.
5441	*/
5442	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5443	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5444	{
5445	uint64_t u64Val;
5446	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5447	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5448	VMXWriteVmcs64(uVmcsField, u64Val);
5449	}
5450
5451	/*
5452	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5453	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5454	*/
5455	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5456	{
5457	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5458	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5459	{
5460	uint64_t u64Val;
5461	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5462	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5463	VMXWriteVmcs64(uVmcsField, u64Val);
5464	}
5465	}
5466
5467	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5468	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5469	}
5470
5471	ASMSetFlags(fEFlags);
5472	return rc;
5473	}
5474
5475
5476	/**
5477	* Copies the shadow VMCS to the nested-guest VMCS.
5478	*
5479	* @returns VBox status code.
5480	* @param pVCpu The cross context virtual CPU structure.
5481	* @param pVmcsInfo The VMCS info. object.
5482	*
5483	* @remarks Called with interrupts disabled.
5484	*/
5485	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5486	{
5487	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5488	PVM pVM = pVCpu->CTX_SUFF(pVM);
5489	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5490
5491	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5492	if (RT_SUCCESS(rc))
5493	{
5494	/*
5495	* Copy guest read/write fields from the shadow VMCS.
5496	* Guest read-only fields cannot be modified, so no need to copy them.
5497	*
5498	* We don't check for VMREAD failures here for performance reasons and
5499	* because they are not expected to fail, barring irrecoverable conditions
5500	* like hardware errors.
5501	*/
5502	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5503	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5504	{
5505	uint64_t u64Val;
5506	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5507	VMXReadVmcs64(uVmcsField, &u64Val);
5508	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5509	}
5510
5511	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5512	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5513	}
5514	return rc;
5515	}
5516
5517
5518	/**
5519	* Enables VMCS shadowing for the given VMCS info. object.
5520	*
5521	* @param pVCpu The cross context virtual CPU structure.
5522	* @param pVmcsInfo The VMCS info. object.
5523	*
5524	* @remarks No-long-jump zone!!!
5525	*/
5526	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5527	{
5528	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5529
5530	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5531	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5532	{
5533	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5534	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5535	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5536	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5537	AssertRC(rc);
5538
5539	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5540	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5541	Log4Func(("Enabled\n"));
5542	}
5543	}
5544
5545
5546	/**
5547	* Disables VMCS shadowing for the given VMCS info. object.
5548	*
5549	* @param pVCpu The cross context virtual CPU structure.
5550	* @param pVmcsInfo The VMCS info. object.
5551	*
5552	* @remarks No-long-jump zone!!!
5553	*/
5554	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5555	{
5556	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5557
5558	/*
5559	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5560	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5561	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5562	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5563	*
5564	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5565	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5566	*/
5567	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5568	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5569	{
5570	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5571	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5572	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5573	AssertRC(rc);
5574
5575	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5576	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5577	Log4Func(("Disabled\n"));
5578	}
5579	}
5580	#endif
5581
5582
5583	/**
5584	* Exports the guest hardware-virtualization state.
5585	*
5586	* @returns VBox status code.
5587	* @param pVCpu The cross context virtual CPU structure.
5588	* @param pVmxTransient The VMX-transient structure.
5589	*
5590	* @remarks No-long-jump zone!!!
5591	*/
5592	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5593	{
5594	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5595	{
5596	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5597	/*
5598	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5599	* VMCS shadowing.
5600	*/
5601	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5602	{
5603	/*
5604	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5605	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5606	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5607	*
5608	* We check for VMX root mode here in case the guest executes VMXOFF without
5609	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5610	* not clear the current VMCS pointer.
5611	*/
5612	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5613	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5614	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5615	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5616	{
5617	/* Paranoia. */
5618	Assert(!pVmxTransient->fIsNestedGuest);
5619
5620	/*
5621	* For performance reasons, also check if the guest hypervisor's current VMCS
5622	* was newly loaded or modified before copying it to the shadow VMCS.
5623	*/
5624	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5625	{
5626	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5627	AssertRCReturn(rc, rc);
5628	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5629	}
5630	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5631	}
5632	else
5633	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5634	}
5635	#else
5636	NOREF(pVmxTransient);
5637	#endif
5638	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5639	}
5640	return VINF_SUCCESS;
5641	}
5642
5643
5644	/**
5645	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5646	*
5647	* The guest FPU state is always pre-loaded hence we don't need to bother about
5648	* sharing FPU related CR0 bits between the guest and host.
5649	*
5650	* @returns VBox status code.
5651	* @param pVCpu The cross context virtual CPU structure.
5652	* @param pVmxTransient The VMX-transient structure.
5653	*
5654	* @remarks No-long-jump zone!!!
5655	*/
5656	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5657	{
5658	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5659	{
5660	PVM pVM = pVCpu->CTX_SUFF(pVM);
5661	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5662
5663	/*
5664	* Figure out fixed CR0 bits in VMX operation.
5665	*/
5666	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5667	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5668	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5669	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5670	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5671	else
5672	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5673
5674	if (!pVmxTransient->fIsNestedGuest)
5675	{
5676	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5677	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5678	uint64_t const u64ShadowCr0 = u64GuestCr0;
5679	Assert(!RT_HI_U32(u64GuestCr0));
5680
5681	/*
5682	* Setup VT-x's view of the guest CR0.
5683	*/
5684	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5685	if (pVM->hm.s.fNestedPaging)
5686	{
5687	if (CPUMIsGuestPagingEnabled(pVCpu))
5688	{
5689	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5690	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5691	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5692	}
5693	else
5694	{
5695	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5696	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5697	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5698	}
5699
5700	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5701	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5702	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5703	}
5704	else
5705	{
5706	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5707	u64GuestCr0 \|= X86_CR0_WP;
5708	}
5709
5710	/*
5711	* Guest FPU bits.
5712	*
5713	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5714	* using CR0.TS.
5715	*
5716	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5717	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5718	*/
5719	u64GuestCr0 \|= X86_CR0_NE;
5720
5721	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5722	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5723
5724	/*
5725	* Update exception intercepts.
5726	*/
5727	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5728	if (pVmcsInfo->RealMode.fRealOnV86Active)
5729	{
5730	Assert(PDMVmmDevHeapIsEnabled(pVM));
5731	Assert(pVM->hm.s.vmx.pRealModeTSS);
5732	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5733	}
5734	else
5735	{
5736	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5737	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5738	if (fInterceptMF)
5739	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5740	}
5741
5742	/* Additional intercepts for debugging, define these yourself explicitly. */
5743	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5744	uXcptBitmap \|= 0
5745	\| RT_BIT(X86_XCPT_BP)
5746	\| RT_BIT(X86_XCPT_DE)
5747	\| RT_BIT(X86_XCPT_NM)
5748	\| RT_BIT(X86_XCPT_TS)
5749	\| RT_BIT(X86_XCPT_UD)
5750	\| RT_BIT(X86_XCPT_NP)
5751	\| RT_BIT(X86_XCPT_SS)
5752	\| RT_BIT(X86_XCPT_GP)
5753	\| RT_BIT(X86_XCPT_PF)
5754	\| RT_BIT(X86_XCPT_MF)
5755	;
5756	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5757	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5758	#endif
5759	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5760	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5761	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5762
5763	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5764	u64GuestCr0 \|= fSetCr0;
5765	u64GuestCr0 &= fZapCr0;
5766	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5767
5768	/* Commit the CR0 and related fields to the guest VMCS. */
5769	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5770	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5771	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5772	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5773	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5774	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5775	AssertRCReturn(rc, rc);
5776
5777	/* Update our caches. */
5778	pVmcsInfo->u32ProcCtls = uProcCtls;
5779	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5780
5781	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5782	}
5783	else
5784	{
5785	/*
5786	* With nested-guests, we may have extended the guest/host mask here since we
5787	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5788	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5789	* originally supplied. We must copy those bits from the nested-guest CR0 into
5790	* the nested-guest CR0 read-shadow.
5791	*/
5792	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5793	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5794	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5795	Assert(!RT_HI_U32(u64GuestCr0));
5796	Assert(u64GuestCr0 & X86_CR0_NE);
5797
5798	/*
5799	* Apply the hardware specified fixed CR0 bits and enable caching.
5800	* Note! We could be altering our VMX emulation's fixed bits. We thus
5801	* need to re-apply them while importing CR0.
5802	*/
5803	u64GuestCr0 \|= fSetCr0;
5804	u64GuestCr0 &= fZapCr0;
5805	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5806
5807	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5808	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5809	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5810	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5811	AssertRCReturn(rc, rc);
5812
5813	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5814	}
5815
5816	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5817	}
5818
5819	return VINF_SUCCESS;
5820	}
5821
5822
5823	/**
5824	* Exports the guest control registers (CR3, CR4) into the guest-state area
5825	* in the VMCS.
5826	*
5827	* @returns VBox strict status code.
5828	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5829	* without unrestricted guest access and the VMMDev is not presently
5830	* mapped (e.g. EFI32).
5831	*
5832	* @param pVCpu The cross context virtual CPU structure.
5833	* @param pVmxTransient The VMX-transient structure.
5834	*
5835	* @remarks No-long-jump zone!!!
5836	*/
5837	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5838	{
5839	int rc = VINF_SUCCESS;
5840	PVM pVM = pVCpu->CTX_SUFF(pVM);
5841
5842	/*
5843	* Guest CR2.
5844	* It's always loaded in the assembler code. Nothing to do here.
5845	*/
5846
5847	/*
5848	* Guest CR3.
5849	*/
5850	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5851	{
5852	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5853
5854	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5855	if (pVM->hm.s.fNestedPaging)
5856	{
5857	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5858	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5859
5860	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5861	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5862	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5863	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5864
5865	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5866	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5867	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5868
5869	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5870	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5871	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5872	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5873	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5874	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5875	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5876
5877	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5878	AssertRCReturn(rc, rc);
5879
5880	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5881	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5882	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5883	{
5884	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5885	if (CPUMIsGuestInPAEModeEx(pCtx))
5886	{
5887	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5888	AssertRCReturn(rc, rc);
5889	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5890	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5891	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5892	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5893	AssertRCReturn(rc, rc);
5894	}
5895
5896	/*
5897	* The guest's view of its CR3 is unblemished with nested paging when the
5898	* guest is using paging or we have unrestricted guest execution to handle
5899	* the guest when it's not using paging.
5900	*/
5901	GCPhysGuestCr3 = pCtx->cr3;
5902	}
5903	else
5904	{
5905	/*
5906	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5907	* thinks it accesses physical memory directly, we use our identity-mapped
5908	* page table to map guest-linear to guest-physical addresses. EPT takes care
5909	* of translating it to host-physical addresses.
5910	*/
5911	RTGCPHYS GCPhys;
5912	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5913
5914	/* We obtain it here every time as the guest could have relocated this PCI region. */
5915	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5916	if (RT_SUCCESS(rc))
5917	{ /* likely */ }
5918	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5919	{
5920	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5921	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5922	}
5923	else
5924	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5925
5926	GCPhysGuestCr3 = GCPhys;
5927	}
5928
5929	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5930	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5931	AssertRCReturn(rc, rc);
5932	}
5933	else
5934	{
5935	/* Non-nested paging case, just use the hypervisor's CR3. */
5936	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5937
5938	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5939	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5940	AssertRCReturn(rc, rc);
5941	}
5942
5943	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5944	}
5945
5946	/*
5947	* Guest CR4.
5948	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5949	*/
5950	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5951	{
5952	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5953	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5954
5955	/*
5956	* Figure out fixed CR4 bits in VMX operation.
5957	*/
5958	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5959	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5960	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5961
5962	/*
5963	* With nested-guests, we may have extended the guest/host mask here (since we
5964	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5965	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5966	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5967	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5968	*/
5969	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5970	uint64_t u64GuestCr4 = pCtx->cr4;
5971	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5972	? pCtx->cr4
5973	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5974	Assert(!RT_HI_U32(u64GuestCr4));
5975
5976	/*
5977	* Setup VT-x's view of the guest CR4.
5978	*
5979	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5980	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5981	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5982	*
5983	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5984	*/
5985	if (pVmcsInfo->RealMode.fRealOnV86Active)
5986	{
5987	Assert(pVM->hm.s.vmx.pRealModeTSS);
5988	Assert(PDMVmmDevHeapIsEnabled(pVM));
5989	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5990	}
5991
5992	if (pVM->hm.s.fNestedPaging)
5993	{
5994	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5995	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5996	{
5997	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5998	u64GuestCr4 \|= X86_CR4_PSE;
5999	/* Our identity mapping is a 32-bit page directory. */
6000	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
6001	}
6002	/* else use guest CR4.*/
6003	}
6004	else
6005	{
6006	Assert(!pVmxTransient->fIsNestedGuest);
6007
6008	/*
6009	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
6010	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
6011	*/
6012	switch (pVCpu->hm.s.enmShadowMode)
6013	{
6014	case PGMMODE_REAL: /* Real-mode. */
6015	case PGMMODE_PROTECTED: /* Protected mode without paging. */
6016	case PGMMODE_32_BIT: /* 32-bit paging. */
6017	{
6018	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
6019	break;
6020	}
6021
6022	case PGMMODE_PAE: /* PAE paging. */
6023	case PGMMODE_PAE_NX: /* PAE paging with NX. */
6024	{
6025	u64GuestCr4 \|= X86_CR4_PAE;
6026	break;
6027	}
6028
6029	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
6030	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
6031	#ifdef VBOX_ENABLE_64_BITS_GUESTS
6032	break;
6033	#endif
6034	default:
6035	AssertFailed();
6036	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6037	}
6038	}
6039
6040	/*
6041	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
6042	* Note! For nested-guests, we could be altering our VMX emulation's
6043	* fixed bits. We thus need to re-apply them while importing CR4.
6044	*/
6045	u64GuestCr4 \|= fSetCr4;
6046	u64GuestCr4 &= fZapCr4;
6047
6048	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
6049	/** @todo Fix to 64-bit when we drop 32-bit. */
6050	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
6051	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
6052	AssertRCReturn(rc, rc);
6053
6054	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
6055	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6056
6057	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
6058
6059	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
6060	}
6061	return rc;
6062	}
6063
6064
6065	/**
6066	* Exports the guest debug registers into the guest-state area in the VMCS.
6067	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
6068	*
6069	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
6070	*
6071	* @returns VBox status code.
6072	* @param pVCpu The cross context virtual CPU structure.
6073	* @param pVmxTransient The VMX-transient structure.
6074	*
6075	* @remarks No-long-jump zone!!!
6076	*/
6077	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6078	{
6079	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6080
6081	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
6082	* stepping. */
6083	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6084	if (pVmxTransient->fIsNestedGuest)
6085	{
6086	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
6087	AssertRCReturn(rc, rc);
6088	return VINF_SUCCESS;
6089	}
6090
6091	#ifdef VBOX_STRICT
6092	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
6093	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
6094	{
6095	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
6096	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
6097	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
6098	}
6099	#endif
6100
6101	bool fSteppingDB = false;
6102	bool fInterceptMovDRx = false;
6103	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6104	if (pVCpu->hm.s.fSingleInstruction)
6105	{
6106	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
6107	PVM pVM = pVCpu->CTX_SUFF(pVM);
6108	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
6109	{
6110	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
6111	Assert(fSteppingDB == false);
6112	}
6113	else
6114	{
6115	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
6116	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
6117	pVCpu->hm.s.fClearTrapFlag = true;
6118	fSteppingDB = true;
6119	}
6120	}
6121
6122	uint32_t u32GuestDr7;
6123	if ( fSteppingDB
6124	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
6125	{
6126	/*
6127	* Use the combined guest and host DRx values found in the hypervisor register set
6128	* because the hypervisor debugger has breakpoints active or someone is single stepping
6129	* on the host side without a monitor trap flag.
6130	*
6131	* Note! DBGF expects a clean DR6 state before executing guest code.
6132	*/
6133	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6134	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6135	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
6136	{
6137	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6138	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
6139	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
6140	}
6141	else
6142	#endif
6143	if (!CPUMIsHyperDebugStateActive(pVCpu))
6144	{
6145	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6146	Assert(CPUMIsHyperDebugStateActive(pVCpu));
6147	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6148	}
6149
6150	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
6151	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
6152	pVCpu->hm.s.fUsingHyperDR7 = true;
6153	fInterceptMovDRx = true;
6154	}
6155	else
6156	{
6157	/*
6158	* If the guest has enabled debug registers, we need to load them prior to
6159	* executing guest code so they'll trigger at the right time.
6160	*/
6161	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6162	{
6163	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6164	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6165	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
6166	{
6167	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6168	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
6169	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
6170	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6171	}
6172	else
6173	#endif
6174	if (!CPUMIsGuestDebugStateActive(pVCpu))
6175	{
6176	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6177	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6178	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6179	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6180	}
6181	Assert(!fInterceptMovDRx);
6182	}
6183	/*
6184	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6185	* must intercept #DB in order to maintain a correct DR6 guest value, and
6186	* because we need to intercept it to prevent nested #DBs from hanging the
6187	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6188	*/
6189	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6190	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
6191	&& !CPUMIsGuestDebugStateActive(pVCpu))
6192	#else
6193	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6194	#endif
6195	{
6196	fInterceptMovDRx = true;
6197	}
6198
6199	/* Update DR7 with the actual guest value. */
6200	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6201	pVCpu->hm.s.fUsingHyperDR7 = false;
6202	}
6203
6204	if (fInterceptMovDRx)
6205	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6206	else
6207	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6208
6209	/*
6210	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6211	* monitor-trap flag and update our cache.
6212	*/
6213	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6214	{
6215	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6216	AssertRCReturn(rc2, rc2);
6217	pVmcsInfo->u32ProcCtls = uProcCtls;
6218	}
6219
6220	/*
6221	* Update guest DR7.
6222	*/
6223	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6224	AssertRCReturn(rc, rc);
6225
6226	/*
6227	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6228	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6229	*
6230	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6231	*/
6232	if (fSteppingDB)
6233	{
6234	Assert(pVCpu->hm.s.fSingleInstruction);
6235	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6236
6237	uint32_t fIntrState = 0;
6238	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6239	AssertRCReturn(rc, rc);
6240
6241	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6242	{
6243	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6244	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6245	AssertRCReturn(rc, rc);
6246	}
6247	}
6248
6249	return VINF_SUCCESS;
6250	}
6251
6252
6253	#ifdef VBOX_STRICT
6254	/**
6255	* Strict function to validate segment registers.
6256	*
6257	* @param pVCpu The cross context virtual CPU structure.
6258	* @param pVmcsInfo The VMCS info. object.
6259	*
6260	* @remarks Will import guest CR0 on strict builds during validation of
6261	* segments.
6262	*/
6263	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6264	{
6265	/*
6266	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6267	*
6268	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6269	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6270	* unusable bit and doesn't change the guest-context value.
6271	*/
6272	PVM pVM = pVCpu->CTX_SUFF(pVM);
6273	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6274	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6275	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6276	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6277	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6278	{
6279	/* Protected mode checks */
6280	/* CS */
6281	Assert(pCtx->cs.Attr.n.u1Present);
6282	Assert(!(pCtx->cs.Attr.u & 0xf00));
6283	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6284	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6285	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6286	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6287	\|\| (pCtx->cs.Attr.n.u1Granularity));
6288	/* CS cannot be loaded with NULL in protected mode. */
6289	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6290	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6291	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6292	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6293	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6294	else
6295	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6296	/* SS */
6297	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6298	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6299	if ( !(pCtx->cr0 & X86_CR0_PE)
6300	\|\| pCtx->cs.Attr.n.u4Type == 3)
6301	{
6302	Assert(!pCtx->ss.Attr.n.u2Dpl);
6303	}
6304	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6305	{
6306	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6307	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6308	Assert(pCtx->ss.Attr.n.u1Present);
6309	Assert(!(pCtx->ss.Attr.u & 0xf00));
6310	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6311	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6312	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6313	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6314	\|\| (pCtx->ss.Attr.n.u1Granularity));
6315	}
6316	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6317	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6318	{
6319	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6320	Assert(pCtx->ds.Attr.n.u1Present);
6321	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6322	Assert(!(pCtx->ds.Attr.u & 0xf00));
6323	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6324	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6325	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6326	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6327	\|\| (pCtx->ds.Attr.n.u1Granularity));
6328	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6329	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6330	}
6331	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6332	{
6333	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6334	Assert(pCtx->es.Attr.n.u1Present);
6335	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6336	Assert(!(pCtx->es.Attr.u & 0xf00));
6337	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6338	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6339	\|\| !(pCtx->es.Attr.n.u1Granularity));
6340	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6341	\|\| (pCtx->es.Attr.n.u1Granularity));
6342	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6343	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6344	}
6345	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6346	{
6347	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6348	Assert(pCtx->fs.Attr.n.u1Present);
6349	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6350	Assert(!(pCtx->fs.Attr.u & 0xf00));
6351	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6352	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6353	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6354	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6355	\|\| (pCtx->fs.Attr.n.u1Granularity));
6356	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6357	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6358	}
6359	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6360	{
6361	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6362	Assert(pCtx->gs.Attr.n.u1Present);
6363	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6364	Assert(!(pCtx->gs.Attr.u & 0xf00));
6365	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6366	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6367	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6368	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6369	\|\| (pCtx->gs.Attr.n.u1Granularity));
6370	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6371	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6372	}
6373	/* 64-bit capable CPUs. */
6374	# if HC_ARCH_BITS == 64
6375	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6376	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6377	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6378	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6379	# endif
6380	}
6381	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6382	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6383	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6384	{
6385	/* Real and v86 mode checks. */
6386	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6387	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6388	if (pVmcsInfo->RealMode.fRealOnV86Active)
6389	{
6390	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6391	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6392	}
6393	else
6394	{
6395	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6396	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6397	}
6398
6399	/* CS */
6400	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6401	Assert(pCtx->cs.u32Limit == 0xffff);
6402	Assert(u32CSAttr == 0xf3);
6403	/* SS */
6404	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6405	Assert(pCtx->ss.u32Limit == 0xffff);
6406	Assert(u32SSAttr == 0xf3);
6407	/* DS */
6408	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6409	Assert(pCtx->ds.u32Limit == 0xffff);
6410	Assert(u32DSAttr == 0xf3);
6411	/* ES */
6412	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6413	Assert(pCtx->es.u32Limit == 0xffff);
6414	Assert(u32ESAttr == 0xf3);
6415	/* FS */
6416	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6417	Assert(pCtx->fs.u32Limit == 0xffff);
6418	Assert(u32FSAttr == 0xf3);
6419	/* GS */
6420	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6421	Assert(pCtx->gs.u32Limit == 0xffff);
6422	Assert(u32GSAttr == 0xf3);
6423	/* 64-bit capable CPUs. */
6424	# if HC_ARCH_BITS == 64
6425	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6426	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6427	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6428	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6429	# endif
6430	}
6431	}
6432	#endif /* VBOX_STRICT */
6433
6434
6435	/**
6436	* Exports a guest segment register into the guest-state area in the VMCS.
6437	*
6438	* @returns VBox status code.
6439	* @param pVCpu The cross context virtual CPU structure.
6440	* @param pVmcsInfo The VMCS info. object.
6441	* @param iSegReg The segment register number (X86_SREG_XXX).
6442	* @param pSelReg Pointer to the segment selector.
6443	*
6444	* @remarks No-long-jump zone!!!
6445	*/
6446	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6447	{
6448	Assert(iSegReg < X86_SREG_COUNT);
6449	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6450	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6451	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6452	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6453
6454	uint32_t u32Access = pSelReg->Attr.u;
6455	if (pVmcsInfo->RealMode.fRealOnV86Active)
6456	{
6457	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6458	u32Access = 0xf3;
6459	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6460	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6461	RT_NOREF_PV(pVCpu);
6462	}
6463	else
6464	{
6465	/*
6466	* The way to differentiate between whether this is really a null selector or was just
6467	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6468	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6469	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6470	* NULL selectors loaded in protected-mode have their attribute as 0.
6471	*/
6472	if (!u32Access)
6473	u32Access = X86DESCATTR_UNUSABLE;
6474	}
6475
6476	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6477	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6478	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6479
6480	/*
6481	* Commit it to the VMCS.
6482	*/
6483	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6484	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6485	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6486	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6487	AssertRCReturn(rc, rc);
6488	return rc;
6489	}
6490
6491
6492	/**
6493	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6494	* area in the VMCS.
6495	*
6496	* @returns VBox status code.
6497	* @param pVCpu The cross context virtual CPU structure.
6498	* @param pVmxTransient The VMX-transient structure.
6499	*
6500	* @remarks Will import guest CR0 on strict builds during validation of
6501	* segments.
6502	* @remarks No-long-jump zone!!!
6503	*/
6504	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6505	{
6506	int rc = VERR_INTERNAL_ERROR_5;
6507	PVM pVM = pVCpu->CTX_SUFF(pVM);
6508	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6509	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6510
6511	/*
6512	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6513	*/
6514	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6515	{
6516	#ifdef VBOX_WITH_REM
6517	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6518	{
6519	Assert(!pVmxTransient->fIsNestedGuest);
6520	Assert(pVM->hm.s.vmx.pRealModeTSS);
6521	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6522	if ( pVmcsInfo->fWasInRealMode
6523	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6524	{
6525	/*
6526	* Notify the recompiler must flush its code-cache as the guest -may-
6527	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6528	*/
6529	REMFlushTBs(pVM);
6530	Log4Func(("Switch to protected mode detected!\n"));
6531	pVmcsInfo->fWasInRealMode = false;
6532	}
6533	}
6534	#endif
6535	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6536	{
6537	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6538	if (pVmcsInfo->RealMode.fRealOnV86Active)
6539	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6540	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6541	AssertRCReturn(rc, rc);
6542	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6543	}
6544
6545	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6546	{
6547	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6548	if (pVmcsInfo->RealMode.fRealOnV86Active)
6549	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6550	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6551	AssertRCReturn(rc, rc);
6552	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6553	}
6554
6555	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6556	{
6557	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6558	if (pVmcsInfo->RealMode.fRealOnV86Active)
6559	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6560	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6561	AssertRCReturn(rc, rc);
6562	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6563	}
6564
6565	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6566	{
6567	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6568	if (pVmcsInfo->RealMode.fRealOnV86Active)
6569	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6570	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6571	AssertRCReturn(rc, rc);
6572	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6573	}
6574
6575	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6576	{
6577	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6578	if (pVmcsInfo->RealMode.fRealOnV86Active)
6579	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6580	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6581	AssertRCReturn(rc, rc);
6582	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6583	}
6584
6585	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6586	{
6587	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6588	if (pVmcsInfo->RealMode.fRealOnV86Active)
6589	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6590	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6591	AssertRCReturn(rc, rc);
6592	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6593	}
6594
6595	#ifdef VBOX_STRICT
6596	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6597	#endif
6598	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6599	pCtx->cs.Attr.u));
6600	}
6601
6602	/*
6603	* Guest TR.
6604	*/
6605	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6606	{
6607	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6608
6609	/*
6610	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6611	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6612	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6613	*/
6614	uint16_t u16Sel;
6615	uint32_t u32Limit;
6616	uint64_t u64Base;
6617	uint32_t u32AccessRights;
6618	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6619	{
6620	u16Sel = pCtx->tr.Sel;
6621	u32Limit = pCtx->tr.u32Limit;
6622	u64Base = pCtx->tr.u64Base;
6623	u32AccessRights = pCtx->tr.Attr.u;
6624	}
6625	else
6626	{
6627	Assert(!pVmxTransient->fIsNestedGuest);
6628	Assert(pVM->hm.s.vmx.pRealModeTSS);
6629	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6630
6631	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6632	RTGCPHYS GCPhys;
6633	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6634	AssertRCReturn(rc, rc);
6635
6636	X86DESCATTR DescAttr;
6637	DescAttr.u = 0;
6638	DescAttr.n.u1Present = 1;
6639	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6640
6641	u16Sel = 0;
6642	u32Limit = HM_VTX_TSS_SIZE;
6643	u64Base = GCPhys;
6644	u32AccessRights = DescAttr.u;
6645	}
6646
6647	/* Validate. */
6648	Assert(!(u16Sel & RT_BIT(2)));
6649	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6650	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6651	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6652	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6653	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6654	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6655	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6656	Assert( (u32Limit & 0xfff) == 0xfff
6657	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6658	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6659	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6660
6661	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6662	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6663	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6664	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6665	AssertRCReturn(rc, rc);
6666
6667	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6668	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6669	}
6670
6671	/*
6672	* Guest GDTR.
6673	*/
6674	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6675	{
6676	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6677
6678	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6679	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6680	AssertRCReturn(rc, rc);
6681
6682	/* Validate. */
6683	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6684
6685	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6686	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6687	}
6688
6689	/*
6690	* Guest LDTR.
6691	*/
6692	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6693	{
6694	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6695
6696	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6697	uint32_t u32Access;
6698	if ( !pVmxTransient->fIsNestedGuest
6699	&& !pCtx->ldtr.Attr.u)
6700	u32Access = X86DESCATTR_UNUSABLE;
6701	else
6702	u32Access = pCtx->ldtr.Attr.u;
6703
6704	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6705	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6706	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6707	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6708	AssertRCReturn(rc, rc);
6709
6710	/* Validate. */
6711	if (!(u32Access & X86DESCATTR_UNUSABLE))
6712	{
6713	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6714	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6715	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6716	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6717	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6718	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6719	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6720	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6721	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6722	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6723	}
6724
6725	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6726	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6727	}
6728
6729	/*
6730	* Guest IDTR.
6731	*/
6732	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6733	{
6734	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6735
6736	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6737	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6738	AssertRCReturn(rc, rc);
6739
6740	/* Validate. */
6741	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6742
6743	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6744	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6745	}
6746
6747	return VINF_SUCCESS;
6748	}
6749
6750
6751	/**
6752	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6753	* areas.
6754	*
6755	* These MSRs will automatically be loaded to the host CPU on every successful
6756	* VM-entry and stored from the host CPU on every successful VM-exit.
6757	*
6758	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6759	* actual host MSR values are not- updated here for performance reasons. See
6760	* hmR0VmxExportHostMsrs().
6761	*
6762	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6763	*
6764	* @returns VBox status code.
6765	* @param pVCpu The cross context virtual CPU structure.
6766	* @param pVmxTransient The VMX-transient structure.
6767	*
6768	* @remarks No-long-jump zone!!!
6769	*/
6770	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6771	{
6772	AssertPtr(pVCpu);
6773	AssertPtr(pVmxTransient);
6774
6775	PVM pVM = pVCpu->CTX_SUFF(pVM);
6776	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6777
6778	/*
6779	* MSRs that we use the auto-load/store MSR area in the VMCS.
6780	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6781	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6782	*
6783	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6784	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6785	* emulation, nothing to do here.
6786	*/
6787	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6788	{
6789	if ( !pVmxTransient->fIsNestedGuest
6790	&& pVM->hm.s.fAllow64BitGuests)
6791	{
6792	#if HC_ARCH_BITS == 32
6793	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6794	Assert(!pVmxTransient->fIsNestedGuest);
6795
6796	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6797	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6798	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6799	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6800	AssertRCReturn(rc, rc);
6801	#endif
6802	}
6803	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6804	}
6805
6806	/*
6807	* Guest Sysenter MSRs.
6808	*/
6809	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6810	{
6811	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6812
6813	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6814	{
6815	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6816	AssertRCReturn(rc, rc);
6817	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6818	}
6819
6820	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6821	{
6822	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6823	AssertRCReturn(rc, rc);
6824	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6825	}
6826
6827	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6828	{
6829	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6830	AssertRCReturn(rc, rc);
6831	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6832	}
6833	}
6834
6835	/*
6836	* Guest/host EFER MSR.
6837	*/
6838	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6839	{
6840	/* Whether we are using the VMCS to swap the EFER MSR must have been
6841	determined earlier while exporting VM-entry/VM-exit controls. */
6842	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6843	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6844
6845	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6846	{
6847	/*
6848	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6849	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6850	*/
6851	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6852	{
6853	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6854	AssertRCReturn(rc, rc);
6855	}
6856	else
6857	{
6858	/*
6859	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6860	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6861	*/
6862	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6863	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6864	AssertRCReturn(rc, rc);
6865	}
6866	}
6867	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6868	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6869
6870	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6871	}
6872
6873	/*
6874	* Other MSRs.
6875	* Speculation Control (R/W).
6876	*/
6877	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6878	{
6879	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6880	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6881	{
6882	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6883	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6884	AssertRCReturn(rc, rc);
6885	}
6886	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6887	}
6888
6889	return VINF_SUCCESS;
6890	}
6891
6892
6893	/**
6894	* Selects up the appropriate function to run guest code.
6895	*
6896	* @returns VBox status code.
6897	* @param pVCpu The cross context virtual CPU structure.
6898	* @param pVmxTransient The VMX-transient structure.
6899	*
6900	* @remarks No-long-jump zone!!!
6901	*/
6902	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6903	{
6904	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6905	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6906
6907	if (CPUMIsGuestInLongModeEx(pCtx))
6908	{
6909	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6910	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6911	#endif
6912	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6913	#if HC_ARCH_BITS == 32
6914	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6915	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6916	{
6917	#ifdef VBOX_STRICT
6918	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6919	{
6920	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6921	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6922	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6923	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6924	("fCtxChanged=%#RX64\n", fCtxChanged));
6925	}
6926	#endif
6927	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6928
6929	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6930	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6931	pVmcsInfo->fSwitchedTo64on32 = true;
6932	Log4Func(("Selected 64-bit switcher\n"));
6933	}
6934	#else
6935	/* 64-bit host. */
6936	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6937	#endif
6938	}
6939	else
6940	{
6941	/* Guest is not in long mode, use the 32-bit handler. */
6942	#if HC_ARCH_BITS == 32
6943	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6944	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6945	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6946	{
6947	# ifdef VBOX_STRICT
6948	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6949	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6950	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6951	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6952	("fCtxChanged=%#RX64\n", fCtxChanged));
6953	# endif
6954	}
6955	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6956	/*
6957	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6958	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6959	* switcher flag now because we know the guest is in a sane state where it's safe
6960	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6961	* the much faster 32-bit switcher again.
6962	*/
6963	if (!pVmcsInfo->fSwitchedTo64on32)
6964	{
6965	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6966	Log4Func(("Selected 32-bit switcher\n"));
6967	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6968	}
6969	else
6970	{
6971	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6972	if ( pVmcsInfo->RealMode.fRealOnV86Active
6973	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6974	{
6975	pVmcsInfo->fSwitchedTo64on32 = false;
6976	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6977	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6978	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6979	\| HM_CHANGED_HOST_CONTEXT);
6980	Log4Func(("Selected 32-bit switcher (safe)\n"));
6981	}
6982	}
6983	# else
6984	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6985	# endif
6986	#else
6987	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6988	#endif
6989	}
6990	Assert(pVmcsInfo->pfnStartVM);
6991	return VINF_SUCCESS;
6992	}
6993
6994
6995	/**
6996	* Wrapper for running the guest code in VT-x.
6997	*
6998	* @returns VBox status code, no informational status codes.
6999	* @param pVCpu The cross context virtual CPU structure.
7000	* @param pVmxTransient The VMX-transient structure.
7001	*
7002	* @remarks No-long-jump zone!!!
7003	*/
7004	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
7005	{
7006	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
7007	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7008	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
7009
7010	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
7011
7012	/*
7013	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
7014	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
7015	* callee-saved and thus the need for this XMM wrapper.
7016	*
7017	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
7018	*/
7019	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7020	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
7021	PVM pVM = pVCpu->CTX_SUFF(pVM);
7022	#ifdef VBOX_WITH_KERNEL_USING_XMM
7023	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
7024	#else
7025	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
7026	#endif
7027	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
7028	return rc;
7029	}
7030
7031
7032	/**
7033	* Reports world-switch error and dumps some useful debug info.
7034	*
7035	* @param pVCpu The cross context virtual CPU structure.
7036	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
7037	* @param pVmxTransient The VMX-transient structure (only
7038	* exitReason updated).
7039	*/
7040	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
7041	{
7042	Assert(pVCpu);
7043	Assert(pVmxTransient);
7044	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7045
7046	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
7047	switch (rcVMRun)
7048	{
7049	case VERR_VMX_INVALID_VMXON_PTR:
7050	AssertFailed();
7051	break;
7052	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
7053	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
7054	{
7055	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
7056	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
7057	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
7058	AssertRC(rc);
7059
7060	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7061	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
7062	Cannot do it here as we may have been long preempted. */
7063
7064	#ifdef VBOX_STRICT
7065	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7066	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
7067	pVmxTransient->uExitReason));
7068	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
7069	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
7070	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
7071	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
7072	else
7073	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
7074	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
7075	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
7076
7077	/* VMX control bits. */
7078	uint32_t u32Val;
7079	uint64_t u64Val;
7080	RTHCUINTREG uHCReg;
7081	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
7082	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
7083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
7084	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
7085	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
7086	{
7087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
7088	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
7089	}
7090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
7091	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
7092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
7093	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
7094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
7095	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
7096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
7097	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
7098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
7099	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
7100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
7101	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
7102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
7103	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
7104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
7105	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
7106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7107	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
7108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7109	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
7110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
7111	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
7112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
7113	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
7114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
7115	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
7116	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7117	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7118	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7119	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7120	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7121	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7122	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7123	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7124	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7125	{
7126	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7127	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7128	}
7129
7130	/* Guest bits. */
7131	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
7132	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
7133	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
7134	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
7135	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
7136	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
7137	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
7138	{
7139	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
7140	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
7141	}
7142
7143	/* Host bits. */
7144	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
7145	Log4(("Host CR0 %#RHr\n", uHCReg));
7146	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
7147	Log4(("Host CR3 %#RHr\n", uHCReg));
7148	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
7149	Log4(("Host CR4 %#RHr\n", uHCReg));
7150
7151	RTGDTR HostGdtr;
7152	PCX86DESCHC pDesc;
7153	ASMGetGDTR(&HostGdtr);
7154	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
7155	Log4(("Host CS %#08x\n", u32Val));
7156	if (u32Val < HostGdtr.cbGdt)
7157	{
7158	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7159	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
7160	}
7161
7162	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
7163	Log4(("Host DS %#08x\n", u32Val));
7164	if (u32Val < HostGdtr.cbGdt)
7165	{
7166	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7167	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
7168	}
7169
7170	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
7171	Log4(("Host ES %#08x\n", u32Val));
7172	if (u32Val < HostGdtr.cbGdt)
7173	{
7174	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7175	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
7176	}
7177
7178	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
7179	Log4(("Host FS %#08x\n", u32Val));
7180	if (u32Val < HostGdtr.cbGdt)
7181	{
7182	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7183	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
7184	}
7185
7186	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
7187	Log4(("Host GS %#08x\n", u32Val));
7188	if (u32Val < HostGdtr.cbGdt)
7189	{
7190	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7191	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
7192	}
7193
7194	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
7195	Log4(("Host SS %#08x\n", u32Val));
7196	if (u32Val < HostGdtr.cbGdt)
7197	{
7198	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7199	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
7200	}
7201
7202	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
7203	Log4(("Host TR %#08x\n", u32Val));
7204	if (u32Val < HostGdtr.cbGdt)
7205	{
7206	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7207	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
7208	}
7209
7210	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
7211	Log4(("Host TR Base %#RHv\n", uHCReg));
7212	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
7213	Log4(("Host GDTR Base %#RHv\n", uHCReg));
7214	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
7215	Log4(("Host IDTR Base %#RHv\n", uHCReg));
7216	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
7217	Log4(("Host SYSENTER CS %#08x\n", u32Val));
7218	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
7219	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
7220	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
7221	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
7222	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
7223	Log4(("Host RSP %#RHv\n", uHCReg));
7224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
7225	Log4(("Host RIP %#RHv\n", uHCReg));
7226	# if HC_ARCH_BITS == 64
7227	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
7228	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
7229	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
7230	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
7231	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
7232	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
7233	# endif
7234	#endif /* VBOX_STRICT */
7235	break;
7236	}
7237
7238	default:
7239	/* Impossible */
7240	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
7241	break;
7242	}
7243	}
7244
7245
7246	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
7247	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
7248	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
7249	# endif
7250
7251	/**
7252	* Initialize the VMCS-Read cache.
7253	*
7254	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
7255	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
7256	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
7257	* (those that have a 32-bit FULL & HIGH part).
7258	*
7259	* @param pVCpu The cross context virtual CPU structure.
7260	*/
7261	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
7262	{
7263	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
7264	do { \
7265	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
7266	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
7267	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
7268	++cReadFields; \
7269	} while (0)
7270
7271	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7272	uint32_t cReadFields = 0;
7273
7274	/*
7275	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
7276	* and serve to indicate exceptions to the rules.
7277	*/
7278
7279	/* Guest-natural selector base fields. */
7280	#if 0
7281	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
7282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
7283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
7284	#endif
7285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
7286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
7287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
7288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
7289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
7290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
7291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
7292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
7293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
7294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
7295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
7296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
7297	#if 0
7298	/* Unused natural width guest-state fields. */
7299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7301	#endif
7302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7304
7305	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7306	these 64-bit fields (using "FULL" and "HIGH" fields). */
7307	#if 0
7308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7317	#endif
7318
7319	/* Natural width guest-state fields. */
7320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7322
7323	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7324	{
7325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7326	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7327	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7328	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7329	}
7330	else
7331	{
7332	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7333	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7334	}
7335
7336	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7337	}
7338
7339
7340	/**
7341	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7342	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7343	* darwin, running 64-bit guests).
7344	*
7345	* @returns VBox status code.
7346	* @param pVCpu The cross context virtual CPU structure.
7347	* @param idxField The VMCS field encoding.
7348	* @param u64Val 16, 32 or 64-bit value.
7349	*/
7350	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7351	{
7352	AssertPtr(pVCpu);
7353	int rc;
7354	switch (idxField)
7355	{
7356	/*
7357	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7358	*/
7359	/* 64-bit Control fields. */
7360	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7361	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7362	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7363	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7364	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7365	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7366	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7367	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7368	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7369	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7370	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7371	case VMX_VMCS64_CTRL_EPTP_FULL:
7372	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7373	/* 64-bit Guest-state fields. */
7374	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7375	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7376	case VMX_VMCS64_GUEST_PAT_FULL:
7377	case VMX_VMCS64_GUEST_EFER_FULL:
7378	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7379	case VMX_VMCS64_GUEST_PDPTE0_FULL:
7380	case VMX_VMCS64_GUEST_PDPTE1_FULL:
7381	case VMX_VMCS64_GUEST_PDPTE2_FULL:
7382	case VMX_VMCS64_GUEST_PDPTE3_FULL:
7383	/* 64-bit Host-state fields. */
7384	case VMX_VMCS64_HOST_PAT_FULL:
7385	case VMX_VMCS64_HOST_EFER_FULL:
7386	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7387	{
7388	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7389	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7390	break;
7391	}
7392
7393	/*
7394	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7395	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7396	*/
7397	/* Natural-width Guest-state fields. */
7398	case VMX_VMCS_GUEST_CR3:
7399	case VMX_VMCS_GUEST_ES_BASE:
7400	case VMX_VMCS_GUEST_CS_BASE:
7401	case VMX_VMCS_GUEST_SS_BASE:
7402	case VMX_VMCS_GUEST_DS_BASE:
7403	case VMX_VMCS_GUEST_FS_BASE:
7404	case VMX_VMCS_GUEST_GS_BASE:
7405	case VMX_VMCS_GUEST_LDTR_BASE:
7406	case VMX_VMCS_GUEST_TR_BASE:
7407	case VMX_VMCS_GUEST_GDTR_BASE:
7408	case VMX_VMCS_GUEST_IDTR_BASE:
7409	case VMX_VMCS_GUEST_RSP:
7410	case VMX_VMCS_GUEST_RIP:
7411	case VMX_VMCS_GUEST_SYSENTER_ESP:
7412	case VMX_VMCS_GUEST_SYSENTER_EIP:
7413	{
7414	if (!(RT_HI_U32(u64Val)))
7415	{
7416	/* If this field is 64-bit, VT-x will zero out the top bits. */
7417	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7418	}
7419	else
7420	{
7421	/* Assert that only the 32->64 switcher case should ever come here. */
7422	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7423	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7424	}
7425	break;
7426	}
7427
7428	default:
7429	{
7430	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7431	pVCpu->hm.s.u32HMError = idxField;
7432	rc = VERR_INVALID_PARAMETER;
7433	break;
7434	}
7435	}
7436	AssertRCReturn(rc, rc);
7437	return rc;
7438	}
7439
7440
7441	/**
7442	* Queue up a VMWRITE by using the VMCS write cache.
7443	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7444	*
7445	* @param pVCpu The cross context virtual CPU structure.
7446	* @param idxField The VMCS field encoding.
7447	* @param u64Val 16, 32 or 64-bit value.
7448	*/
7449	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7450	{
7451	AssertPtr(pVCpu);
7452	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7453
7454	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7455	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7456
7457	/* Make sure there are no duplicates. */
7458	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7459	{
7460	if (pCache->Write.aField[i] == idxField)
7461	{
7462	pCache->Write.aFieldVal[i] = u64Val;
7463	return VINF_SUCCESS;
7464	}
7465	}
7466
7467	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7468	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7469	pCache->Write.cValidEntries++;
7470	return VINF_SUCCESS;
7471	}
7472	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7473
7474
7475	/**
7476	* Sets up the usage of TSC-offsetting and updates the VMCS.
7477	*
7478	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7479	* VMX-preemption timer.
7480	*
7481	* @returns VBox status code.
7482	* @param pVCpu The cross context virtual CPU structure.
7483	* @param pVmxTransient The VMX-transient structure.
7484	*
7485	* @remarks No-long-jump zone!!!
7486	*/
7487	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7488	{
7489	bool fOffsettedTsc;
7490	bool fParavirtTsc;
7491	uint64_t uTscOffset;
7492	PVM pVM = pVCpu->CTX_SUFF(pVM);
7493	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7494
7495	if (pVM->hm.s.vmx.fUsePreemptTimer)
7496	{
7497	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7498
7499	/* Make sure the returned values have sane upper and lower boundaries. */
7500	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7501	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7502	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7503	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7504
7505	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7506	* preemption timers here. We probably need to clamp the preemption timer,
7507	* after converting the timer value to the host. */
7508	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7509	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7510	AssertRC(rc);
7511	}
7512	else
7513	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7514
7515	if (fParavirtTsc)
7516	{
7517	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7518	information before every VM-entry, hence disable it for performance sake. */
7519	#if 0
7520	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7521	AssertRC(rc);
7522	#endif
7523	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7524	}
7525
7526	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
7527	if ( fOffsettedTsc
7528	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7529	{
7530	if (pVmxTransient->fIsNestedGuest)
7531	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7532	if (pVmcsInfo->u64TscOffset != uTscOffset)
7533	{
7534	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
7535	AssertRC(rc);
7536	pVmcsInfo->u64TscOffset = uTscOffset;
7537	}
7538
7539	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
7540	{
7541	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
7542	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7543	AssertRC(rc);
7544	pVmcsInfo->u32ProcCtls = uProcCtls;
7545	}
7546	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
7547	}
7548	else
7549	{
7550	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7551	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
7552	{
7553	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
7554	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7555	AssertRC(rc);
7556	pVmcsInfo->u32ProcCtls = uProcCtls;
7557	}
7558	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
7559	}
7560	}
7561
7562
7563	/**
7564	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7565	* VM-exit interruption info type.
7566	*
7567	* @returns The IEM exception flags.
7568	* @param uVector The event vector.
7569	* @param uVmxEventType The VMX event type.
7570	*
7571	* @remarks This function currently only constructs flags required for
7572	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7573	* and CR2 aspects of an exception are not included).
7574	*/
7575	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7576	{
7577	uint32_t fIemXcptFlags;
7578	switch (uVmxEventType)
7579	{
7580	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7581	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7582	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7583	break;
7584
7585	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7586	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7587	break;
7588
7589	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7590	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7591	break;
7592
7593	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7594	{
7595	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7596	if (uVector == X86_XCPT_BP)
7597	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7598	else if (uVector == X86_XCPT_OF)
7599	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7600	else
7601	{
7602	fIemXcptFlags = 0;
7603	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7604	}
7605	break;
7606	}
7607
7608	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7609	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7610	break;
7611
7612	default:
7613	fIemXcptFlags = 0;
7614	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7615	break;
7616	}
7617	return fIemXcptFlags;
7618	}
7619
7620
7621	/**
7622	* Sets an event as a pending event to be injected into the guest.
7623	*
7624	* @param pVCpu The cross context virtual CPU structure.
7625	* @param u32IntInfo The VM-entry interruption-information field.
7626	* @param cbInstr The VM-entry instruction length in bytes (for software
7627	* interrupts, exceptions and privileged software
7628	* exceptions).
7629	* @param u32ErrCode The VM-entry exception error code.
7630	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7631	* page-fault.
7632	*/
7633	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7634	RTGCUINTPTR GCPtrFaultAddress)
7635	{
7636	Assert(!pVCpu->hm.s.Event.fPending);
7637	pVCpu->hm.s.Event.fPending = true;
7638	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7639	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7640	pVCpu->hm.s.Event.cbInstr = cbInstr;
7641	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7642	}
7643
7644
7645	/**
7646	* Sets an external interrupt as pending-for-injection into the VM.
7647	*
7648	* @param pVCpu The cross context virtual CPU structure.
7649	* @param u8Interrupt The external interrupt vector.
7650	*/
7651	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7652	{
7653	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7654	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7655	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7657	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7658	}
7659
7660
7661	/**
7662	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7663	*
7664	* @param pVCpu The cross context virtual CPU structure.
7665	*/
7666	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7667	{
7668	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7669	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7670	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7672	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7673	}
7674
7675
7676	/**
7677	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7678	*
7679	* @param pVCpu The cross context virtual CPU structure.
7680	*/
7681	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7682	{
7683	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7684	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7685	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7686	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7687	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7688	}
7689
7690
7691	/**
7692	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7693	*
7694	* @param pVCpu The cross context virtual CPU structure.
7695	*/
7696	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7697	{
7698	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7699	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7700	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7702	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7703	}
7704
7705
7706	/**
7707	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7708	*
7709	* @param pVCpu The cross context virtual CPU structure.
7710	*/
7711	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7712	{
7713	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7714	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7715	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7716	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7717	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7718	}
7719
7720
7721	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7722	/**
7723	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7724	*
7725	* @param pVCpu The cross context virtual CPU structure.
7726	* @param u32ErrCode The error code for the general-protection exception.
7727	*/
7728	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7729	{
7730	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7731	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7732	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7733	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7734	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7735	}
7736
7737
7738	/**
7739	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7740	*
7741	* @param pVCpu The cross context virtual CPU structure.
7742	* @param u32ErrCode The error code for the stack exception.
7743	*/
7744	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7745	{
7746	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7747	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7748	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7750	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7751	}
7752	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7753
7754
7755	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7756	{
7757	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7758
7759	/*
7760	* If VT-x marks the segment as unusable, most other bits remain undefined:
7761	* - For CS the L, D and G bits have meaning.
7762	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7763	* - For the remaining data segments no bits are defined.
7764	*
7765	* The present bit and the unusable bit has been observed to be set at the
7766	* same time (the selector was supposed to be invalid as we started executing
7767	* a V8086 interrupt in ring-0).
7768	*
7769	* What should be important for the rest of the VBox code, is that the P bit is
7770	* cleared. Some of the other VBox code recognizes the unusable bit, but
7771	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7772	* safe side here, we'll strip off P and other bits we don't care about. If
7773	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7774	*
7775	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7776	*/
7777	#ifdef VBOX_STRICT
7778	uint32_t const uAttr = pSelReg->Attr.u;
7779	#endif
7780
7781	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7782	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7783	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7784
7785	#ifdef VBOX_STRICT
7786	VMMRZCallRing3Disable(pVCpu);
7787	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7788	# ifdef DEBUG_bird
7789	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7790	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7791	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7792	# endif
7793	VMMRZCallRing3Enable(pVCpu);
7794	NOREF(uAttr);
7795	#endif
7796	RT_NOREF2(pVCpu, idxSel);
7797	}
7798
7799
7800	/**
7801	* Imports a guest segment register from the current VMCS into the guest-CPU
7802	* context.
7803	*
7804	* @returns VBox status code.
7805	* @param pVCpu The cross context virtual CPU structure.
7806	* @param iSegReg The segment register number (X86_SREG_XXX).
7807	*
7808	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7809	* do not log!
7810	*/
7811	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7812	{
7813	Assert(iSegReg < X86_SREG_COUNT);
7814
7815	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7816	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7817	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7818	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7819	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7820	#else
7821	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7822	#endif
7823	uint64_t u64Base;
7824	uint32_t u32Sel, u32Limit, u32Attr;
7825	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7826	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7827	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7828	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7829	if (RT_SUCCESS(rc))
7830	{
7831	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7832	pSelReg->Sel = u32Sel;
7833	pSelReg->ValidSel = u32Sel;
7834	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7835	pSelReg->u32Limit = u32Limit;
7836	pSelReg->u64Base = u64Base;
7837	pSelReg->Attr.u = u32Attr;
7838	if (u32Attr & X86DESCATTR_UNUSABLE)
7839	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7840	}
7841	return rc;
7842	}
7843
7844
7845	/**
7846	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7847	*
7848	* @returns VBox status code.
7849	* @param pVCpu The cross context virtual CPU structure.
7850	*
7851	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7852	* do not log!
7853	*/
7854	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7855	{
7856	uint64_t u64Base;
7857	uint32_t u32Sel, u32Limit, u32Attr;
7858	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7859	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7860	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7861	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7862	if (RT_SUCCESS(rc))
7863	{
7864	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7865	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7866	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7867	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7868	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7869	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7870	if (u32Attr & X86DESCATTR_UNUSABLE)
7871	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7872	}
7873	return rc;
7874	}
7875
7876
7877	/**
7878	* Imports the guest TR from the current VMCS into the guest-CPU context.
7879	*
7880	* @returns VBox status code.
7881	* @param pVCpu The cross context virtual CPU structure.
7882	*
7883	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7884	* do not log!
7885	*/
7886	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7887	{
7888	uint32_t u32Sel, u32Limit, u32Attr;
7889	uint64_t u64Base;
7890	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7891	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7892	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7893	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7894	AssertRCReturn(rc, rc);
7895
7896	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7897	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7898	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7899	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7900	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7901	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7902	/* TR is the only selector that can never be unusable. */
7903	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7904	return VINF_SUCCESS;
7905	}
7906
7907
7908	/**
7909	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7910	*
7911	* @returns VBox status code.
7912	* @param pVCpu The cross context virtual CPU structure.
7913	*
7914	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7915	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7916	* instead!!!
7917	*/
7918	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7919	{
7920	uint64_t u64Val;
7921	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7922	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7923	{
7924	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7925	if (RT_SUCCESS(rc))
7926	{
7927	pCtx->rip = u64Val;
7928	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7929	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7930	}
7931	return rc;
7932	}
7933	return VINF_SUCCESS;
7934	}
7935
7936
7937	/**
7938	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7939	*
7940	* @returns VBox status code.
7941	* @param pVCpu The cross context virtual CPU structure.
7942	* @param pVmcsInfo The VMCS info. object.
7943	*
7944	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7945	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7946	* instead!!!
7947	*/
7948	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7949	{
7950	uint32_t u32Val;
7951	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7952	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7953	{
7954	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7955	if (RT_SUCCESS(rc))
7956	{
7957	pCtx->eflags.u32 = u32Val;
7958
7959	/* Restore eflags for real-on-v86-mode hack. */
7960	if (pVmcsInfo->RealMode.fRealOnV86Active)
7961	{
7962	pCtx->eflags.Bits.u1VM = 0;
7963	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7964	}
7965	}
7966	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7967	return rc;
7968	}
7969	return VINF_SUCCESS;
7970	}
7971
7972
7973	/**
7974	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7975	* context.
7976	*
7977	* @returns VBox status code.
7978	* @param pVCpu The cross context virtual CPU structure.
7979	* @param pVmcsInfo The VMCS info. object.
7980	*
7981	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7982	* do not log!
7983	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7984	* instead!!!
7985	*/
7986	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7987	{
7988	uint32_t u32Val;
7989	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7990	if (RT_SUCCESS(rc))
7991	{
7992	if (!u32Val)
7993	{
7994	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7995	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7996
7997	CPUMSetGuestNmiBlocking(pVCpu, false);
7998	}
7999	else
8000	{
8001	/*
8002	* We must import RIP here to set our EM interrupt-inhibited state.
8003	* We also import RFLAGS as our code that evaluates pending interrupts
8004	* before VM-entry requires it.
8005	*/
8006	rc = hmR0VmxImportGuestRip(pVCpu);
8007	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8008	if (RT_SUCCESS(rc))
8009	{
8010	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
8011	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
8012	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
8013	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
8014
8015	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8016	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
8017	}
8018	}
8019	}
8020	return rc;
8021	}
8022
8023
8024	/**
8025	* Worker for VMXR0ImportStateOnDemand.
8026	*
8027	* @returns VBox status code.
8028	* @param pVCpu The cross context virtual CPU structure.
8029	* @param pVmcsInfo The VMCS info. object.
8030	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8031	*/
8032	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
8033	{
8034	#define VMXLOCAL_BREAK_RC(a_rc) \
8035	if (RT_SUCCESS(a_rc)) \
8036	{ } \
8037	else \
8038	break
8039
8040	int rc = VINF_SUCCESS;
8041	PVM pVM = pVCpu->CTX_SUFF(pVM);
8042	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8043	uint64_t u64Val;
8044	uint32_t u32Val;
8045
8046	/*
8047	* Note! This is hack to workaround a mysterious BSOD observed with release builds
8048	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8049	* neither are other host platforms.
8050	*
8051	* Committing this temporarily as it prevents BSOD.
8052	*
8053	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8054	*/
8055	#ifdef RT_OS_WINDOWS
8056	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
8057	return VERR_HM_IPE_1;
8058	#endif
8059
8060	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8061
8062	/*
8063	* We disable interrupts to make the updating of the state and in particular
8064	* the fExtrn modification atomic wrt to preemption hooks.
8065	*/
8066	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8067
8068	fWhat &= pCtx->fExtrn;
8069	if (fWhat)
8070	{
8071	do
8072	{
8073	if (fWhat & CPUMCTX_EXTRN_RIP)
8074	{
8075	rc = hmR0VmxImportGuestRip(pVCpu);
8076	VMXLOCAL_BREAK_RC(rc);
8077	}
8078
8079	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8080	{
8081	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8082	VMXLOCAL_BREAK_RC(rc);
8083	}
8084
8085	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8086	{
8087	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8088	VMXLOCAL_BREAK_RC(rc);
8089	}
8090
8091	if (fWhat & CPUMCTX_EXTRN_RSP)
8092	{
8093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8094	VMXLOCAL_BREAK_RC(rc);
8095	pCtx->rsp = u64Val;
8096	}
8097
8098	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8099	{
8100	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8101	if (fWhat & CPUMCTX_EXTRN_CS)
8102	{
8103	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8104	rc \|= hmR0VmxImportGuestRip(pVCpu);
8105	if (fRealOnV86Active)
8106	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8107	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8108	}
8109	if (fWhat & CPUMCTX_EXTRN_SS)
8110	{
8111	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8112	if (fRealOnV86Active)
8113	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8114	}
8115	if (fWhat & CPUMCTX_EXTRN_DS)
8116	{
8117	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8118	if (fRealOnV86Active)
8119	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8120	}
8121	if (fWhat & CPUMCTX_EXTRN_ES)
8122	{
8123	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8124	if (fRealOnV86Active)
8125	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8126	}
8127	if (fWhat & CPUMCTX_EXTRN_FS)
8128	{
8129	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8130	if (fRealOnV86Active)
8131	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8132	}
8133	if (fWhat & CPUMCTX_EXTRN_GS)
8134	{
8135	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8136	if (fRealOnV86Active)
8137	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8138	}
8139	VMXLOCAL_BREAK_RC(rc);
8140	}
8141
8142	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8143	{
8144	if (fWhat & CPUMCTX_EXTRN_LDTR)
8145	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
8146
8147	if (fWhat & CPUMCTX_EXTRN_GDTR)
8148	{
8149	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8150	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8151	pCtx->gdtr.pGdt = u64Val;
8152	pCtx->gdtr.cbGdt = u32Val;
8153	}
8154
8155	/* Guest IDTR. */
8156	if (fWhat & CPUMCTX_EXTRN_IDTR)
8157	{
8158	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8159	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8160	pCtx->idtr.pIdt = u64Val;
8161	pCtx->idtr.cbIdt = u32Val;
8162	}
8163
8164	/* Guest TR. */
8165	if (fWhat & CPUMCTX_EXTRN_TR)
8166	{
8167	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8168	don't need to import that one. */
8169	if (!pVmcsInfo->RealMode.fRealOnV86Active)
8170	rc \|= hmR0VmxImportGuestTr(pVCpu);
8171	}
8172	VMXLOCAL_BREAK_RC(rc);
8173	}
8174
8175	if (fWhat & CPUMCTX_EXTRN_DR7)
8176	{
8177	if (!pVCpu->hm.s.fUsingHyperDR7)
8178	{
8179	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8180	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8181	VMXLOCAL_BREAK_RC(rc);
8182	pCtx->dr[7] = u32Val;
8183	}
8184	}
8185
8186	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8187	{
8188	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8189	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8190	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8191	pCtx->SysEnter.cs = u32Val;
8192	VMXLOCAL_BREAK_RC(rc);
8193	}
8194
8195	#if HC_ARCH_BITS == 64
8196	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8197	{
8198	if ( pVM->hm.s.fAllow64BitGuests
8199	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8200	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8201	}
8202
8203	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8204	{
8205	if ( pVM->hm.s.fAllow64BitGuests
8206	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8207	{
8208	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8209	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8210	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8211	}
8212	}
8213	#endif
8214
8215	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
8216	#if HC_ARCH_BITS == 32
8217	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
8218	#endif
8219	)
8220	{
8221	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8222	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8223	Assert(pMsrs);
8224	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8225	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
8226	for (uint32_t i = 0; i < cMsrs; i++)
8227	{
8228	uint32_t const idMsr = pMsrs[i].u32Msr;
8229	switch (idMsr)
8230	{
8231	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8232	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8233	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8234	#if HC_ARCH_BITS == 32
8235	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8236	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8237	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8238	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8239	#endif
8240	default:
8241	{
8242	pCtx->fExtrn = 0;
8243	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8244	ASMSetFlags(fEFlags);
8245	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8246	return VERR_HM_UNEXPECTED_LD_ST_MSR;
8247	}
8248	}
8249	}
8250	}
8251
8252	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8253	{
8254	uint64_t u64Shadow;
8255	if (fWhat & CPUMCTX_EXTRN_CR0)
8256	{
8257	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8258	* remove when we drop 32-bit host w/ 64-bit host support, see
8259	* @bugref{9180#c39}. */
8260	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8261	#if HC_ARCH_BITS == 32
8262	uint32_t u32Shadow;
8263	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8264	u64Shadow = u32Shadow;
8265	#else
8266	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8267	#endif
8268	VMXLOCAL_BREAK_RC(rc);
8269	u64Val = u32Val;
8270	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8271	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
8272	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8273	/*
8274	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
8275	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8276	*/
8277	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8278	{
8279	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8280	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8281	}
8282	#endif
8283	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8284	CPUMSetGuestCR0(pVCpu, u64Val);
8285	VMMRZCallRing3Enable(pVCpu);
8286	}
8287
8288	if (fWhat & CPUMCTX_EXTRN_CR4)
8289	{
8290	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8291	* remove when we drop 32-bit host w/ 64-bit host support, see
8292	* @bugref{9180#c39}. */
8293	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8294	#if HC_ARCH_BITS == 32
8295	uint32_t u32Shadow;
8296	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8297	u64Shadow = u32Shadow;
8298	#else
8299	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8300	#endif
8301	VMXLOCAL_BREAK_RC(rc);
8302	u64Val = u32Val;
8303	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8304	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
8305	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8306	/*
8307	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
8308	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8309	*/
8310	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8311	{
8312	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8313	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8314	}
8315	#endif
8316	pCtx->cr4 = u64Val;
8317	}
8318
8319	if (fWhat & CPUMCTX_EXTRN_CR3)
8320	{
8321	/* CR0.PG bit changes are always intercepted, so it's up to date. */
8322	if ( pVM->hm.s.vmx.fUnrestrictedGuest
8323	\|\| ( pVM->hm.s.fNestedPaging
8324	&& CPUMIsGuestPagingEnabledEx(pCtx)))
8325	{
8326	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8327	VMXLOCAL_BREAK_RC(rc);
8328	if (pCtx->cr3 != u64Val)
8329	{
8330	pCtx->cr3 = u64Val;
8331	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8332	}
8333
8334	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8335	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8336	if (CPUMIsGuestInPAEModeEx(pCtx))
8337	{
8338	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8339	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8340	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8341	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8342	VMXLOCAL_BREAK_RC(rc);
8343	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8344	}
8345	}
8346	}
8347	}
8348
8349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8350	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
8351	{
8352	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
8353	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
8354	{
8355	Assert(CPUMIsGuestInVmxRootMode(pCtx));
8356	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
8357	VMXLOCAL_BREAK_RC(rc);
8358	}
8359
8360	# if 0
8361	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8362	* VM-exit handlers as parameters. We would handle it differently when using
8363	* the fast path. */
8364	/*
8365	* The hardware virtualization state currently consists of VMCS fields that may be
8366	* modified by execution of the nested-guest (that are not part of the general
8367	* guest state) and is visible to guest software. Hence, it is technically part of
8368	* the guest-CPU state when executing a nested-guest.
8369	*/
8370	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8371	{
8372	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8373	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8374	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8375	VMXLOCAL_BREAK_RC(rc);
8376
8377	/*
8378	* VM-entry can fail due to invalid-guest state, machine-check events and
8379	* MSR loading failures. Other than VM-exit reason and Exit qualification
8380	* all other VMCS fields are left unmodified on VM-entry failure.
8381	*
8382	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8383	*/
8384	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8385	if (!fEntryFailed)
8386	{
8387	/*
8388	* Some notes on VMCS fields that may need importing when the fast path
8389	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8390	*
8391	* Requires fixing up when using hardware-assisted VMX:
8392	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8393	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
8394	* - IDT-vectoring info: Think about this.
8395	* - IDT-vectoring error code: Think about this.
8396	*
8397	* Emulated:
8398	* - Guest-interruptiblity state: Derived from FFs and RIP.
8399	* - Guest pending debug exceptions: Derived from DR6.
8400	* - Guest activity state: Emulated from EM state.
8401	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8402	* - Entry-interrupt info: Emulated, cleared to 0.
8403	*/
8404	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8405	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8406	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8407	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8408	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8409	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8410	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8411	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8412	/** @todo NSTVMX: Save and adjust preemption timer value. */
8413	}
8414
8415	VMXLOCAL_BREAK_RC(rc);
8416	}
8417	# endif
8418	}
8419	#endif
8420	} while (0);
8421
8422	if (RT_SUCCESS(rc))
8423	{
8424	/* Update fExtrn. */
8425	pCtx->fExtrn &= ~fWhat;
8426
8427	/* If everything has been imported, clear the HM keeper bit. */
8428	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8429	{
8430	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8431	Assert(!pCtx->fExtrn);
8432	}
8433	}
8434	}
8435	else
8436	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8437
8438	ASMSetFlags(fEFlags);
8439
8440	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8441
8442	if (RT_SUCCESS(rc))
8443	{ /* likely */ }
8444	else
8445	return rc;
8446
8447	/*
8448	* Honor any pending CR3 updates.
8449	*
8450	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8451	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8452	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8453	*
8454	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8455	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8456	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8457	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8458	*
8459	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8460	*/
8461	if (VMMRZCallRing3IsEnabled(pVCpu))
8462	{
8463	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8464	{
8465	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8466	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8467	}
8468
8469	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8470	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8471
8472	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8473	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8474	}
8475
8476	return VINF_SUCCESS;
8477	#undef VMXLOCAL_BREAK_RC
8478	}
8479
8480
8481	/**
8482	* Saves the guest state from the VMCS into the guest-CPU context.
8483	*
8484	* @returns VBox status code.
8485	* @param pVCpu The cross context virtual CPU structure.
8486	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8487	*/
8488	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8489	{
8490	AssertPtr(pVCpu);
8491	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8492	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8493	}
8494
8495
8496	/**
8497	* Check per-VM and per-VCPU force flag actions that require us to go back to
8498	* ring-3 for one reason or another.
8499	*
8500	* @returns Strict VBox status code (i.e. informational status codes too)
8501	* @retval VINF_SUCCESS if we don't have any actions that require going back to
8502	* ring-3.
8503	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8504	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8505	* interrupts)
8506	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8507	* all EMTs to be in ring-3.
8508	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8509	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8510	* to the EM loop.
8511	*
8512	* @param pVCpu The cross context virtual CPU structure.
8513	* @param fStepping Whether we are single-stepping the guest using the
8514	* hypervisor debugger.
8515	*/
8516	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8517	{
8518	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8519
8520	/*
8521	* Update pending interrupts into the APIC's IRR.
8522	*/
8523	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8524	APICUpdatePendingInterrupts(pVCpu);
8525
8526	/*
8527	* Anything pending? Should be more likely than not if we're doing a good job.
8528	*/
8529	PVM pVM = pVCpu->CTX_SUFF(pVM);
8530	if ( !fStepping
8531	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8532	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8533	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8534	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8535	return VINF_SUCCESS;
8536
8537	/* Pending PGM C3 sync. */
8538	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8539	{
8540	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8541	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8542	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8543	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8544	if (rcStrict2 != VINF_SUCCESS)
8545	{
8546	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8547	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8548	return rcStrict2;
8549	}
8550	}
8551
8552	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8553	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8554	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8555	{
8556	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8557	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8558	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8559	return rc2;
8560	}
8561
8562	/* Pending VM request packets, such as hardware interrupts. */
8563	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8564	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8565	{
8566	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
8567	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8568	return VINF_EM_PENDING_REQUEST;
8569	}
8570
8571	/* Pending PGM pool flushes. */
8572	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8573	{
8574	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
8575	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8576	return VINF_PGM_POOL_FLUSH_PENDING;
8577	}
8578
8579	/* Pending DMA requests. */
8580	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8581	{
8582	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
8583	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8584	return VINF_EM_RAW_TO_R3;
8585	}
8586
8587	return VINF_SUCCESS;
8588	}
8589
8590
8591	/**
8592	* Converts any TRPM trap into a pending HM event. This is typically used when
8593	* entering from ring-3 (not longjmp returns).
8594	*
8595	* @param pVCpu The cross context virtual CPU structure.
8596	*/
8597	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8598	{
8599	Assert(TRPMHasTrap(pVCpu));
8600	Assert(!pVCpu->hm.s.Event.fPending);
8601
8602	uint8_t uVector;
8603	TRPMEVENT enmTrpmEvent;
8604	RTGCUINT uErrCode;
8605	RTGCUINTPTR GCPtrFaultAddress;
8606	uint8_t cbInstr;
8607
8608	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8609	AssertRC(rc);
8610
8611	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
8612	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
8613	if (enmTrpmEvent == TRPM_TRAP)
8614	{
8615	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
8616	* generated using INT1 (ICEBP). */
8617	switch (uVector)
8618	{
8619	case X86_XCPT_NMI:
8620	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8621	break;
8622
8623	case X86_XCPT_BP:
8624	case X86_XCPT_OF:
8625	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8626	break;
8627
8628	case X86_XCPT_PF:
8629	case X86_XCPT_DF:
8630	case X86_XCPT_TS:
8631	case X86_XCPT_NP:
8632	case X86_XCPT_SS:
8633	case X86_XCPT_GP:
8634	case X86_XCPT_AC:
8635	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
8636	RT_FALL_THRU();
8637	default:
8638	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8639	break;
8640	}
8641	}
8642	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
8643	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8644	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
8645	{
8646	switch (uVector)
8647	{
8648	case X86_XCPT_BP:
8649	case X86_XCPT_OF:
8650	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8651	break;
8652
8653	default:
8654	Assert(uVector == X86_XCPT_DB);
8655	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
8656	break;
8657	}
8658	}
8659	else
8660	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
8661
8662	rc = TRPMResetTrap(pVCpu);
8663	AssertRC(rc);
8664	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8665	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8666
8667	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8668	}
8669
8670
8671	/**
8672	* Converts the pending HM event into a TRPM trap.
8673	*
8674	* @param pVCpu The cross context virtual CPU structure.
8675	*/
8676	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8677	{
8678	Assert(pVCpu->hm.s.Event.fPending);
8679
8680	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8681	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
8682	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
8683	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
8684
8685	/* If a trap was already pending, we did something wrong! */
8686	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8687
8688	/** @todo Use HMVmxEventToTrpmEventType() later. */
8689	TRPMEVENT enmTrapType;
8690	switch (uVectorType)
8691	{
8692	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
8693	enmTrapType = TRPM_HARDWARE_INT;
8694	break;
8695
8696	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
8697	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
8698	enmTrapType = TRPM_TRAP;
8699	break;
8700
8701	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
8702	Assert(uVector == X86_XCPT_DB);
8703	enmTrapType = TRPM_SOFTWARE_INT;
8704	break;
8705
8706	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
8707	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8708	enmTrapType = TRPM_SOFTWARE_INT;
8709	break;
8710
8711	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
8712	enmTrapType = TRPM_SOFTWARE_INT;
8713	break;
8714
8715	default:
8716	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
8717	enmTrapType = TRPM_32BIT_HACK;
8718	break;
8719	}
8720
8721	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8722
8723	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8724	AssertRC(rc);
8725
8726	if (fErrorCodeValid)
8727	TRPMSetErrorCode(pVCpu, uErrorCode);
8728
8729	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8730	&& uVector == X86_XCPT_PF)
8731	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8732	else if (enmTrapType == TRPM_SOFTWARE_INT)
8733	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8734
8735	/* We're now done converting the pending event. */
8736	pVCpu->hm.s.Event.fPending = false;
8737	}
8738
8739
8740	/**
8741	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8742	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8743	*
8744	* @param pVCpu The cross context virtual CPU structure.
8745	* @param pVmcsInfo The VMCS info. object.
8746	*/
8747	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8748	{
8749	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8750	{
8751	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8752	{
8753	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8754	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8755	AssertRC(rc);
8756	}
8757	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8758	}
8759
8760
8761	/**
8762	* Clears the interrupt-window exiting control in the VMCS.
8763	*
8764	* @param pVmcsInfo The VMCS info. object.
8765	*/
8766	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8767	{
8768	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8769	{
8770	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8771	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8772	}
8773	return VINF_SUCCESS;
8774	}
8775
8776
8777	/**
8778	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8779	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8780	*
8781	* @param pVCpu The cross context virtual CPU structure.
8782	* @param pVmcsInfo The VMCS info. object.
8783	*/
8784	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8785	{
8786	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8787	{
8788	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8789	{
8790	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8791	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8792	AssertRC(rc);
8793	Log4Func(("Setup NMI-window exiting\n"));
8794	}
8795	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8796	}
8797
8798
8799	/**
8800	* Clears the NMI-window exiting control in the VMCS.
8801	*
8802	* @param pVmcsInfo The VMCS info. object.
8803	*/
8804	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8805	{
8806	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8807	{
8808	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8809	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8810	}
8811	return VINF_SUCCESS;
8812	}
8813
8814
8815	/**
8816	* Does the necessary state syncing before returning to ring-3 for any reason
8817	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8818	*
8819	* @returns VBox status code.
8820	* @param pVCpu The cross context virtual CPU structure.
8821	* @param fImportState Whether to import the guest state from the VMCS back
8822	* to the guest-CPU context.
8823	*
8824	* @remarks No-long-jmp zone!!!
8825	*/
8826	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8827	{
8828	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8829	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8830
8831	RTCPUID const idCpu = RTMpCpuId();
8832	Log4Func(("HostCpuId=%u\n", idCpu));
8833
8834	/*
8835	* !!! IMPORTANT !!!
8836	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8837	*/
8838
8839	/* Save the guest state if necessary. */
8840	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8841	if (fImportState)
8842	{
8843	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8844	AssertRCReturn(rc, rc);
8845	}
8846
8847	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8848	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8849	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8850
8851	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8852	#ifdef VBOX_STRICT
8853	if (CPUMIsHyperDebugStateActive(pVCpu))
8854	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8855	#endif
8856	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8857	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8858	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8859
8860	#if HC_ARCH_BITS == 64
8861	/* Restore host-state bits that VT-x only restores partially. */
8862	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8863	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8864	{
8865	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8866	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8867	}
8868	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8869	#endif
8870
8871	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8872	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8873	{
8874	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8875	if (!fImportState)
8876	{
8877	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8878	AssertRCReturn(rc, rc);
8879	}
8880	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8881	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8882	}
8883	else
8884	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8885
8886	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8887	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8888
8889	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8890	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8891	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8892	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8893	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8894	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8895	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8896	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8897	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8898	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8899
8900	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8901
8902	/** @todo This partially defeats the purpose of having preemption hooks.
8903	* The problem is, deregistering the hooks should be moved to a place that
8904	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8905	* context.
8906	*/
8907	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8908	AssertRCReturn(rc, rc);
8909
8910	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8911	/*
8912	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8913	* clear a shadow VMCS before allowing that VMCS to become active on another
8914	* logical processor. We may or may not be importing guest state which clears
8915	* it, so cover for it here.
8916	*
8917	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8918	*/
8919	if ( pVmcsInfo->pvShadowVmcs
8920	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8921	{
8922	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8923	AssertRCReturn(rc, rc);
8924	}
8925
8926	/*
8927	* Flag that we need to re-import the host state if we switch to this VMCS before
8928	* executing guest or nested-guest code.
8929	*/
8930	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8931	#endif
8932
8933	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8934	NOREF(idCpu);
8935	return VINF_SUCCESS;
8936	}
8937
8938
8939	/**
8940	* Leaves the VT-x session.
8941	*
8942	* @returns VBox status code.
8943	* @param pVCpu The cross context virtual CPU structure.
8944	*
8945	* @remarks No-long-jmp zone!!!
8946	*/
8947	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8948	{
8949	HM_DISABLE_PREEMPT(pVCpu);
8950	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8951	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8952	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8953
8954	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8955	and done this from the VMXR0ThreadCtxCallback(). */
8956	if (!pVCpu->hm.s.fLeaveDone)
8957	{
8958	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8959	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8960	pVCpu->hm.s.fLeaveDone = true;
8961	}
8962	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8963
8964	/*
8965	* !!! IMPORTANT !!!
8966	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8967	*/
8968
8969	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8970	/** @todo Deregistering here means we need to VMCLEAR always
8971	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8972	* for calling VMMR0ThreadCtxHookDisable here! */
8973	VMMR0ThreadCtxHookDisable(pVCpu);
8974
8975	/* Leave HM context. This takes care of local init (term). */
8976	int rc = HMR0LeaveCpu(pVCpu);
8977
8978	HM_RESTORE_PREEMPT();
8979	return rc;
8980	}
8981
8982
8983	/**
8984	* Does the necessary state syncing before doing a longjmp to ring-3.
8985	*
8986	* @returns VBox status code.
8987	* @param pVCpu The cross context virtual CPU structure.
8988	*
8989	* @remarks No-long-jmp zone!!!
8990	*/
8991	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8992	{
8993	return hmR0VmxLeaveSession(pVCpu);
8994	}
8995
8996
8997	/**
8998	* Take necessary actions before going back to ring-3.
8999	*
9000	* An action requires us to go back to ring-3. This function does the necessary
9001	* steps before we can safely return to ring-3. This is not the same as longjmps
9002	* to ring-3, this is voluntary and prepares the guest so it may continue
9003	* executing outside HM (recompiler/IEM).
9004	*
9005	* @returns VBox status code.
9006	* @param pVCpu The cross context virtual CPU structure.
9007	* @param rcExit The reason for exiting to ring-3. Can be
9008	* VINF_VMM_UNKNOWN_RING3_CALL.
9009	*/
9010	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
9011	{
9012	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9013
9014	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9015	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
9016	{
9017	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
9018	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
9019	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
9020	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
9021	}
9022
9023	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
9024	VMMRZCallRing3Disable(pVCpu);
9025	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
9026
9027	/*
9028	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
9029	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
9030	*
9031	* This is because execution may continue from ring-3 and we would need to inject
9032	* the event from there (hence place it back in TRPM).
9033	*/
9034	if (pVCpu->hm.s.Event.fPending)
9035	{
9036	hmR0VmxPendingEventToTrpmTrap(pVCpu);
9037	Assert(!pVCpu->hm.s.Event.fPending);
9038
9039	/* Clear the events from the VMCS. */
9040	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
9041	AssertRCReturn(rc, rc);
9042	}
9043	#ifdef VBOX_STRICT
9044	else
9045	{
9046	/*
9047	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
9048	* This can be pretty hard to debug otherwise, interrupts might get injected twice
9049	* occasionally, see @bugref{9180#c42}.
9050	*
9051	* However, if the VM-entry failed, any VM entry-interruption info. field would
9052	* be left unmodified as the event would not have been injected to the guest. In
9053	* such cases, don't assert, we're not going to continue guest execution anyway.
9054	*/
9055	uint32_t uExitReason;
9056	uint32_t uEntryIntInfo;
9057	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9058	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
9059	AssertRC(rc);
9060	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
9061	}
9062	#endif
9063
9064	/*
9065	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
9066	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
9067	* (e.g. TPR below threshold).
9068	*/
9069	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9070	{
9071	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
9072	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
9073	AssertRCReturn(rc, rc);
9074	}
9075
9076	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
9077	and if we're injecting an event we should have a TRPM trap pending. */
9078	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
9079	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
9080	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
9081	#endif
9082
9083	/* Save guest state and restore host state bits. */
9084	int rc = hmR0VmxLeaveSession(pVCpu);
9085	AssertRCReturn(rc, rc);
9086	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
9087
9088	/* Thread-context hooks are unregistered at this point!!! */
9089
9090	/* Sync recompiler state. */
9091	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
9092	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
9093	\| CPUM_CHANGED_LDTR
9094	\| CPUM_CHANGED_GDTR
9095	\| CPUM_CHANGED_IDTR
9096	\| CPUM_CHANGED_TR
9097	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
9098	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
9099	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
9100	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
9101
9102	Assert(!pVCpu->hm.s.fClearTrapFlag);
9103
9104	/* Update the exit-to-ring 3 reason. */
9105	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
9106
9107	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
9108	if ( rcExit != VINF_EM_RAW_INTERRUPT
9109	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9110	{
9111	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
9112	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9113	}
9114
9115	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
9116
9117	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
9118	VMMRZCallRing3RemoveNotification(pVCpu);
9119	VMMRZCallRing3Enable(pVCpu);
9120
9121	return rc;
9122	}
9123
9124
9125	/**
9126	* VMMRZCallRing3() callback wrapper which saves the guest state before we
9127	* longjump to ring-3 and possibly get preempted.
9128	*
9129	* @returns VBox status code.
9130	* @param pVCpu The cross context virtual CPU structure.
9131	* @param enmOperation The operation causing the ring-3 longjump.
9132	* @param pvUser User argument, currently unused, NULL.
9133	*/
9134	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9135	{
9136	RT_NOREF(pvUser);
9137	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9138	{
9139	/*
9140	* !!! IMPORTANT !!!
9141	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9142	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9143	*/
9144	VMMRZCallRing3RemoveNotification(pVCpu);
9145	VMMRZCallRing3Disable(pVCpu);
9146	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9147	RTThreadPreemptDisable(&PreemptState);
9148
9149	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9150	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9151	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9152	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9153
9154	#if HC_ARCH_BITS == 64
9155	/* Restore host-state bits that VT-x only restores partially. */
9156	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9157	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9158	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9159	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9160	#endif
9161
9162	/* Restore the lazy host MSRs as we're leaving VT-x context. */
9163	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9164	hmR0VmxLazyRestoreHostMsrs(pVCpu);
9165
9166	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9167	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9168	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9169
9170	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
9171	cleared as part of importing the guest state above. */
9172	hmR0VmxClearVmcs(pVmcsInfo);
9173
9174	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9175	VMMR0ThreadCtxHookDisable(pVCpu);
9176	HMR0LeaveCpu(pVCpu);
9177	RTThreadPreemptRestore(&PreemptState);
9178	return VINF_SUCCESS;
9179	}
9180
9181	Assert(pVCpu);
9182	Assert(pvUser);
9183	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9184	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9185
9186	VMMRZCallRing3Disable(pVCpu);
9187	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9188
9189	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9190
9191	int rc = hmR0VmxLongJmpToRing3(pVCpu);
9192	AssertRCReturn(rc, rc);
9193
9194	VMMRZCallRing3Enable(pVCpu);
9195	return VINF_SUCCESS;
9196	}
9197
9198
9199	/**
9200	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9201	* stack.
9202	*
9203	* @returns Strict VBox status code (i.e. informational status codes too).
9204	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9205	* @param pVCpu The cross context virtual CPU structure.
9206	* @param uValue The value to push to the guest stack.
9207	*/
9208	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9209	{
9210	/*
9211	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9212	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9213	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9214	*/
9215	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9216	if (pCtx->sp == 1)
9217	return VINF_EM_RESET;
9218	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9219	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9220	AssertRC(rc);
9221	return rc;
9222	}
9223
9224
9225	/**
9226	* Injects an event into the guest upon VM-entry by updating the relevant fields
9227	* in the VM-entry area in the VMCS.
9228	*
9229	* @returns Strict VBox status code (i.e. informational status codes too).
9230	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9231	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9232	*
9233	* @param pVCpu The cross context virtual CPU structure.
9234	* @param pVmxTransient The VMX-transient structure.
9235	* @param pEvent The event being injected.
9236	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
9237	* will be updated if necessary. This cannot not be NULL.
9238	* @param fStepping Whether we're single-stepping guest execution and should
9239	* return VINF_EM_DBG_STEPPED if the event is injected
9240	* directly (registers modified by us, not by hardware on
9241	* VM-entry).
9242	*/
9243	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9244	uint32_t *pfIntrState)
9245	{
9246	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9247	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9248	Assert(pfIntrState);
9249
9250	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9251	uint32_t u32IntInfo = pEvent->u64IntInfo;
9252	uint32_t const u32ErrCode = pEvent->u32ErrCode;
9253	uint32_t const cbInstr = pEvent->cbInstr;
9254	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9255	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9256	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9257
9258	#ifdef VBOX_STRICT
9259	/*
9260	* Validate the error-code-valid bit for hardware exceptions.
9261	* No error codes for exceptions in real-mode.
9262	*
9263	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9264	*/
9265	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9266	&& !CPUMIsGuestInRealModeEx(pCtx))
9267	{
9268	switch (uVector)
9269	{
9270	case X86_XCPT_PF:
9271	case X86_XCPT_DF:
9272	case X86_XCPT_TS:
9273	case X86_XCPT_NP:
9274	case X86_XCPT_SS:
9275	case X86_XCPT_GP:
9276	case X86_XCPT_AC:
9277	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9278	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9279	RT_FALL_THRU();
9280	default:
9281	break;
9282	}
9283	}
9284
9285	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
9286	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9287	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9288	#endif
9289
9290	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9291
9292	/*
9293	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
9294	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9295	* interrupt handler in the (real-mode) guest.
9296	*
9297	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9298	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9299	*/
9300	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9301	{
9302	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9303	{
9304	/*
9305	* For CPUs with unrestricted guest execution enabled and with the guest
9306	* in real-mode, we must not set the deliver-error-code bit.
9307	*
9308	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9309	*/
9310	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9311	}
9312	else
9313	{
9314	PVM pVM = pVCpu->CTX_SUFF(pVM);
9315	Assert(PDMVmmDevHeapIsEnabled(pVM));
9316	Assert(pVM->hm.s.vmx.pRealModeTSS);
9317	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9318
9319	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9320	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9321	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
9322	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
9323	AssertRCReturn(rc2, rc2);
9324
9325	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9326	size_t const cbIdtEntry = sizeof(X86IDTR16);
9327	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9328	{
9329	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9330	if (uVector == X86_XCPT_DF)
9331	return VINF_EM_RESET;
9332
9333	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9334	No error codes for exceptions in real-mode. */
9335	if (uVector == X86_XCPT_GP)
9336	{
9337	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9338	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9339	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9340	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9341	HMEVENT EventXcptDf;
9342	RT_ZERO(EventXcptDf);
9343	EventXcptDf.u64IntInfo = uXcptDfInfo;
9344	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9345	}
9346
9347	/*
9348	* If we're injecting an event with no valid IDT entry, inject a #GP.
9349	* No error codes for exceptions in real-mode.
9350	*
9351	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9352	*/
9353	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9354	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9355	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9356	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9357	HMEVENT EventXcptGp;
9358	RT_ZERO(EventXcptGp);
9359	EventXcptGp.u64IntInfo = uXcptGpInfo;
9360	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9361	}
9362
9363	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9364	uint16_t uGuestIp = pCtx->ip;
9365	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9366	{
9367	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9368	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
9369	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9370	}
9371	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9372	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9373
9374	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9375	X86IDTR16 IdtEntry;
9376	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9377	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9378	AssertRCReturn(rc2, rc2);
9379
9380	/* Construct the stack frame for the interrupt/exception handler. */
9381	VBOXSTRICTRC rcStrict;
9382	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9383	if (rcStrict == VINF_SUCCESS)
9384	{
9385	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9386	if (rcStrict == VINF_SUCCESS)
9387	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9388	}
9389
9390	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
9391	if (rcStrict == VINF_SUCCESS)
9392	{
9393	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
9394	pCtx->rip = IdtEntry.offSel;
9395	pCtx->cs.Sel = IdtEntry.uSel;
9396	pCtx->cs.ValidSel = IdtEntry.uSel;
9397	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9398	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9399	&& uVector == X86_XCPT_PF)
9400	pCtx->cr2 = GCPtrFault;
9401
9402	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
9403	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9404	\| HM_CHANGED_GUEST_RSP);
9405
9406	/*
9407	* If we delivered a hardware exception (other than an NMI) and if there was
9408	* block-by-STI in effect, we should clear it.
9409	*/
9410	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9411	{
9412	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9413	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9414	Log4Func(("Clearing inhibition due to STI\n"));
9415	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9416	}
9417
9418	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9419	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9420
9421	/*
9422	* The event has been truly dispatched to the guest. Mark it as no longer pending so
9423	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9424	*/
9425	pVCpu->hm.s.Event.fPending = false;
9426
9427	/*
9428	* If we eventually support nested-guest execution without unrestricted guest execution,
9429	* we should clear fInterceptEvents here.
9430	*/
9431	Assert(!pVmxTransient->fIsNestedGuest);
9432
9433	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9434	if (fStepping)
9435	rcStrict = VINF_EM_DBG_STEPPED;
9436	}
9437	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9438	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9439	return rcStrict;
9440	}
9441	}
9442
9443	/*
9444	* Validate.
9445	*/
9446	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9447	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9448
9449	/*
9450	* Inject the event into the VMCS.
9451	*/
9452	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9453	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9454	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9455	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9456	AssertRCReturn(rc, rc);
9457
9458	/*
9459	* Update guest CR2 if this is a page-fault.
9460	*/
9461	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9462	&& uVector == X86_XCPT_PF)
9463	pCtx->cr2 = GCPtrFault;
9464
9465	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9466	return VINF_SUCCESS;
9467	}
9468
9469
9470	/**
9471	* Evaluates the event to be delivered to the guest and sets it as the pending
9472	* event.
9473	*
9474	* @returns Strict VBox status code (i.e. informational status codes too).
9475	* @param pVCpu The cross context virtual CPU structure.
9476	* @param pVmxTransient The VMX-transient structure.
9477	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
9478	*/
9479	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9480	{
9481	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9482	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9483	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
9484
9485	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9486	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9487	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9488	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9489	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9490
9491	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9492	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9493	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9494	Assert(!TRPMHasTrap(pVCpu));
9495	Assert(pfIntrState);
9496
9497	*pfIntrState = fIntrState;
9498
9499	/*
9500	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9501	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9502	*/
9503	/** @todo SMI. SMIs take priority over NMIs. */
9504	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9505	{
9506	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9507	if ( !pVCpu->hm.s.Event.fPending
9508	&& !fBlockNmi
9509	&& !fBlockSti
9510	&& !fBlockMovSS)
9511	{
9512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9513	if ( fIsNestedGuest
9514	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9515	return IEMExecVmxVmexitXcptNmi(pVCpu);
9516	#endif
9517	hmR0VmxSetPendingXcptNmi(pVCpu);
9518	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9519	Log4Func(("Pending NMI\n"));
9520	}
9521	else if (!fIsNestedGuest)
9522	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9523	}
9524	/*
9525	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9526	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9527	*/
9528	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
9529	&& !pVCpu->hm.s.fSingleInstruction)
9530	{
9531	Assert(!DBGFIsStepping(pVCpu));
9532	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9533	AssertRCReturn(rc, rc);
9534	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9535	if ( !pVCpu->hm.s.Event.fPending
9536	&& !fBlockInt
9537	&& !fBlockSti
9538	&& !fBlockMovSS)
9539	{
9540	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9541	if ( fIsNestedGuest
9542	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9543	{
9544	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
9545	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9546	return rcStrict;
9547	}
9548	#endif
9549	uint8_t u8Interrupt;
9550	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9551	if (RT_SUCCESS(rc))
9552	{
9553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9554	if ( fIsNestedGuest
9555	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9556	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9557	{
9558	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9559	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9560	return rcStrict;
9561	}
9562	#endif
9563	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9564	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9565	}
9566	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9567	{
9568	if ( !fIsNestedGuest
9569	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9570	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9571	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9572
9573	/*
9574	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9575	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9576	* need to re-set this force-flag here.
9577	*/
9578	}
9579	else
9580	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9581	}
9582	else if (!fIsNestedGuest)
9583	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9584	}
9585
9586	return VINF_SUCCESS;
9587	}
9588
9589
9590	/**
9591	* Injects any pending events into the guest if the guest is in a state to
9592	* receive them.
9593	*
9594	* @returns Strict VBox status code (i.e. informational status codes too).
9595	* @param pVCpu The cross context virtual CPU structure.
9596	* @param pVmxTransient The VMX-transient structure.
9597	* @param fIntrState The VT-x guest-interruptibility state.
9598	* @param fStepping Whether we are single-stepping the guest using the
9599	* hypervisor debugger and should return
9600	* VINF_EM_DBG_STEPPED if the event was dispatched
9601	* directly.
9602	*/
9603	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9604	{
9605	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9606	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9607
9608	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9609	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9610
9611	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9612	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9613	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9614	Assert(!TRPMHasTrap(pVCpu));
9615
9616	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9617	if (pVCpu->hm.s.Event.fPending)
9618	{
9619	/*
9620	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9621	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9622	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9623	*
9624	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9625	*/
9626	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9627	#ifdef VBOX_STRICT
9628	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9629	{
9630	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9631	Assert(!fBlockInt);
9632	Assert(!fBlockSti);
9633	Assert(!fBlockMovSS);
9634	}
9635	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9636	{
9637	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9638	Assert(!fBlockSti);
9639	Assert(!fBlockMovSS);
9640	Assert(!fBlockNmi);
9641	}
9642	#endif
9643	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9644	uIntType));
9645
9646	/*
9647	* Inject the event and get any changes to the guest-interruptibility state.
9648	*
9649	* The guest-interruptibility state may need to be updated if we inject the event
9650	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9651	*/
9652	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9653	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9654
9655	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9656	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9657	else
9658	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9659	}
9660
9661	/*
9662	* Update the guest-interruptibility state.
9663	*
9664	* This is required for the real-on-v86 software interrupt injection case above, as well as
9665	* updates to the guest state from ring-3 or IEM/REM.
9666	*/
9667	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9668	AssertRCReturn(rc, rc);
9669
9670	/*
9671	* There's no need to clear the VM-entry interruption-information field here if we're not
9672	* injecting anything. VT-x clears the valid bit on every VM-exit.
9673	*
9674	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9675	*/
9676
9677	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9678	NOREF(fBlockMovSS); NOREF(fBlockSti);
9679	return rcStrict;
9680	}
9681
9682
9683	/**
9684	* Enters the VT-x session.
9685	*
9686	* @returns VBox status code.
9687	* @param pVCpu The cross context virtual CPU structure.
9688	*/
9689	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9690	{
9691	AssertPtr(pVCpu);
9692	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9693	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9694
9695	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9696	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9697	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9698
9699	#ifdef VBOX_STRICT
9700	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9701	RTCCUINTREG uHostCr4 = ASMGetCR4();
9702	if (!(uHostCr4 & X86_CR4_VMXE))
9703	{
9704	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9705	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9706	}
9707	#endif
9708
9709	/*
9710	* Load the appropriate VMCS as the current and active one.
9711	*/
9712	PVMXVMCSINFO pVmcsInfo;
9713	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9714	if (!fInNestedGuestMode)
9715	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9716	else
9717	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9718	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9719	if (RT_SUCCESS(rc))
9720	{
9721	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9722	pVCpu->hm.s.fLeaveDone = false;
9723	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9724
9725	/*
9726	* Do the EMT scheduled L1D flush here if needed.
9727	*/
9728	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9729	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9730	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9731	hmR0MdsClear();
9732	}
9733	return rc;
9734	}
9735
9736
9737	/**
9738	* The thread-context callback (only on platforms which support it).
9739	*
9740	* @param enmEvent The thread-context event.
9741	* @param pVCpu The cross context virtual CPU structure.
9742	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9743	* @thread EMT(pVCpu)
9744	*/
9745	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9746	{
9747	AssertPtr(pVCpu);
9748	RT_NOREF1(fGlobalInit);
9749
9750	switch (enmEvent)
9751	{
9752	case RTTHREADCTXEVENT_OUT:
9753	{
9754	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9755	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9756	VMCPU_ASSERT_EMT(pVCpu);
9757
9758	/* No longjmps (logger flushes, locks) in this fragile context. */
9759	VMMRZCallRing3Disable(pVCpu);
9760	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9761
9762	/* Restore host-state (FPU, debug etc.) */
9763	if (!pVCpu->hm.s.fLeaveDone)
9764	{
9765	/*
9766	* Do -not- import the guest-state here as we might already be in the middle of importing
9767	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9768	*/
9769	hmR0VmxLeave(pVCpu, false /* fImportState */);
9770	pVCpu->hm.s.fLeaveDone = true;
9771	}
9772
9773	/* Leave HM context, takes care of local init (term). */
9774	int rc = HMR0LeaveCpu(pVCpu);
9775	AssertRC(rc);
9776
9777	/* Restore longjmp state. */
9778	VMMRZCallRing3Enable(pVCpu);
9779	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9780	break;
9781	}
9782
9783	case RTTHREADCTXEVENT_IN:
9784	{
9785	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9786	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9787	VMCPU_ASSERT_EMT(pVCpu);
9788
9789	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9790	VMMRZCallRing3Disable(pVCpu);
9791	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9792
9793	/* Initialize the bare minimum state required for HM. This takes care of
9794	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9795	int rc = hmR0EnterCpu(pVCpu);
9796	AssertRC(rc);
9797	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9798	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9799
9800	/* Load the active VMCS as the current one. */
9801	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9802	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9803	AssertRC(rc);
9804	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9805	pVCpu->hm.s.fLeaveDone = false;
9806
9807	/* Do the EMT scheduled L1D flush if needed. */
9808	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9809	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9810
9811	/* Restore longjmp state. */
9812	VMMRZCallRing3Enable(pVCpu);
9813	break;
9814	}
9815
9816	default:
9817	break;
9818	}
9819	}
9820
9821
9822	/**
9823	* Exports the host state into the VMCS host-state area.
9824	* Sets up the VM-exit MSR-load area.
9825	*
9826	* The CPU state will be loaded from these fields on every successful VM-exit.
9827	*
9828	* @returns VBox status code.
9829	* @param pVCpu The cross context virtual CPU structure.
9830	*
9831	* @remarks No-long-jump zone!!!
9832	*/
9833	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9834	{
9835	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9836
9837	int rc = VINF_SUCCESS;
9838	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9839	{
9840	rc = hmR0VmxExportHostControlRegs();
9841	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9842
9843	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9844	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9845
9846	rc = hmR0VmxExportHostMsrs(pVCpu);
9847	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9848
9849	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9850	}
9851	return rc;
9852	}
9853
9854
9855	/**
9856	* Saves the host state in the VMCS host-state.
9857	*
9858	* @returns VBox status code.
9859	* @param pVCpu The cross context virtual CPU structure.
9860	*
9861	* @remarks No-long-jump zone!!!
9862	*/
9863	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9864	{
9865	AssertPtr(pVCpu);
9866	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9867
9868	/*
9869	* Export the host state here while entering HM context.
9870	* When thread-context hooks are used, we might get preempted and have to re-save the host
9871	* state but most of the time we won't be, so do it here before we disable interrupts.
9872	*/
9873	return hmR0VmxExportHostState(pVCpu);
9874	}
9875
9876
9877	/**
9878	* Exports the guest state into the VMCS guest-state area.
9879	*
9880	* The will typically be done before VM-entry when the guest-CPU state and the
9881	* VMCS state may potentially be out of sync.
9882	*
9883	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9884	* VM-entry controls.
9885	* Sets up the appropriate VMX non-root function to execute guest code based on
9886	* the guest CPU mode.
9887	*
9888	* @returns VBox strict status code.
9889	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9890	* without unrestricted guest execution and the VMMDev is not presently
9891	* mapped (e.g. EFI32).
9892	*
9893	* @param pVCpu The cross context virtual CPU structure.
9894	* @param pVmxTransient The VMX-transient structure.
9895	*
9896	* @remarks No-long-jump zone!!!
9897	*/
9898	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9899	{
9900	AssertPtr(pVCpu);
9901	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9902	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9903
9904	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9905
9906	/*
9907	* Determine real-on-v86 mode.
9908	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9909	*/
9910	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9911	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9912	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9913	pVmcsInfo->RealMode. fRealOnV86Active = false;
9914	else
9915	{
9916	Assert(!pVmxTransient->fIsNestedGuest);
9917	pVmcsInfo->RealMode.fRealOnV86Active = true;
9918	}
9919
9920	/*
9921	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9922	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9923	*/
9924	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9925	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9926	* be a need to evaluate this everytime since I'm pretty sure we intercept
9927	* all guest paging mode changes. */
9928	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9929	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9930
9931	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9932	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9933
9934	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9935	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9936
9937	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9938	if (rcStrict == VINF_SUCCESS)
9939	{ /* likely */ }
9940	else
9941	{
9942	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9943	return rcStrict;
9944	}
9945
9946	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9947	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9948
9949	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9950	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9951
9952	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9953	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9954
9955	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9956	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9957
9958	rc = hmR0VmxExportGuestRip(pVCpu);
9959	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9960	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9961	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9962
9963	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9964	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9965
9966	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9967	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9968	\| HM_CHANGED_GUEST_CR2
9969	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9970	\| HM_CHANGED_GUEST_X87
9971	\| HM_CHANGED_GUEST_SSE_AVX
9972	\| HM_CHANGED_GUEST_OTHER_XSAVE
9973	\| HM_CHANGED_GUEST_XCRx
9974	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9975	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9976	\| HM_CHANGED_GUEST_TSC_AUX
9977	\| HM_CHANGED_GUEST_OTHER_MSRS
9978	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9979
9980	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9981	return rc;
9982	}
9983
9984
9985	/**
9986	* Exports the state shared between the host and guest into the VMCS.
9987	*
9988	* @param pVCpu The cross context virtual CPU structure.
9989	* @param pVmxTransient The VMX-transient structure.
9990	*
9991	* @remarks No-long-jump zone!!!
9992	*/
9993	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9994	{
9995	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9996	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9997
9998	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9999	{
10000	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
10001	AssertRC(rc);
10002	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
10003
10004	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
10005	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
10006	{
10007	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
10008	AssertRC(rc);
10009	}
10010	}
10011
10012	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
10013	{
10014	hmR0VmxLazyLoadGuestMsrs(pVCpu);
10015	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
10016	}
10017
10018	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
10019	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10020	}
10021
10022
10023	/**
10024	* Worker for loading the guest-state bits in the inner VT-x execution loop.
10025	*
10026	* @returns Strict VBox status code (i.e. informational status codes too).
10027	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
10028	* without unrestricted guest execution and the VMMDev is not presently
10029	* mapped (e.g. EFI32).
10030	*
10031	* @param pVCpu The cross context virtual CPU structure.
10032	* @param pVmxTransient The VMX-transient structure.
10033	*
10034	* @remarks No-long-jump zone!!!
10035	*/
10036	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10037	{
10038	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10039	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10040	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10041
10042	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
10043	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
10044	#endif
10045
10046	/*
10047	* For many exits it's only RIP that changes and hence try to export it first
10048	* without going through a lot of change flag checks.
10049	*/
10050	VBOXSTRICTRC rcStrict;
10051	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
10052	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
10053	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
10054	{
10055	rcStrict = hmR0VmxExportGuestRip(pVCpu);
10056	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10057	{ /* likely */}
10058	else
10059	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
10060	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
10061	}
10062	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
10063	{
10064	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
10065	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10066	{ /* likely */}
10067	else
10068	{
10069	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
10070	VBOXSTRICTRC_VAL(rcStrict)));
10071	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10072	return rcStrict;
10073	}
10074	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
10075	}
10076	else
10077	rcStrict = VINF_SUCCESS;
10078
10079	#ifdef VBOX_STRICT
10080	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
10081	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
10082	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
10083	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
10084	("fCtxChanged=%#RX64\n", fCtxChanged));
10085	#endif
10086	return rcStrict;
10087	}
10088
10089
10090	/**
10091	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10092	* and update error record fields accordingly.
10093	*
10094	* @returns VMX_IGS_* error codes.
10095	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10096	* wrong with the guest state.
10097	*
10098	* @param pVCpu The cross context virtual CPU structure.
10099	* @param pVmcsInfo The VMCS info. object.
10100	*
10101	* @remarks This function assumes our cache of the VMCS controls
10102	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10103	*/
10104	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10105	{
10106	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10107	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10108	uError = (err); \
10109	break; \
10110	} else do { } while (0)
10111
10112	int rc;
10113	PVM pVM = pVCpu->CTX_SUFF(pVM);
10114	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10115	uint32_t uError = VMX_IGS_ERROR;
10116	uint32_t u32Val;
10117	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10118
10119	do
10120	{
10121	/*
10122	* CR0.
10123	*/
10124	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10125	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10126	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10127	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10128	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10129	if (fUnrestrictedGuest)
10130	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10131
10132	uint32_t u32GuestCr0;
10133	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10134	AssertRCBreak(rc);
10135	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10136	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10137	if ( !fUnrestrictedGuest
10138	&& (u32GuestCr0 & X86_CR0_PG)
10139	&& !(u32GuestCr0 & X86_CR0_PE))
10140	{
10141	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10142	}
10143
10144	/*
10145	* CR4.
10146	*/
10147	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10148	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10149	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10150
10151	uint32_t u32GuestCr4;
10152	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10153	AssertRCBreak(rc);
10154	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10155	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10156
10157	/*
10158	* IA32_DEBUGCTL MSR.
10159	*/
10160	uint64_t u64Val;
10161	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10162	AssertRCBreak(rc);
10163	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10164	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10165	{
10166	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10167	}
10168	uint64_t u64DebugCtlMsr = u64Val;
10169
10170	#ifdef VBOX_STRICT
10171	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10172	AssertRCBreak(rc);
10173	Assert(u32Val == pVmcsInfo->u32EntryCtls);
10174	#endif
10175	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10176
10177	/*
10178	* RIP and RFLAGS.
10179	*/
10180	uint32_t u32Eflags;
10181	#if HC_ARCH_BITS == 64
10182	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10183	AssertRCBreak(rc);
10184	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10185	if ( !fLongModeGuest
10186	\|\| !pCtx->cs.Attr.n.u1Long)
10187	{
10188	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10189	}
10190	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10191	* must be identical if the "IA-32e mode guest" VM-entry
10192	* control is 1 and CS.L is 1. No check applies if the
10193	* CPU supports 64 linear-address bits. */
10194
10195	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10196	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10197	AssertRCBreak(rc);
10198	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10199	VMX_IGS_RFLAGS_RESERVED);
10200	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10201	u32Eflags = u64Val;
10202	#else
10203	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10204	AssertRCBreak(rc);
10205	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10206	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10207	#endif
10208
10209	if ( fLongModeGuest
10210	\|\| ( fUnrestrictedGuest
10211	&& !(u32GuestCr0 & X86_CR0_PE)))
10212	{
10213	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10214	}
10215
10216	uint32_t u32EntryInfo;
10217	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10218	AssertRCBreak(rc);
10219	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10220	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10221	{
10222	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10223	}
10224
10225	/*
10226	* 64-bit checks.
10227	*/
10228	#if HC_ARCH_BITS == 64
10229	if (fLongModeGuest)
10230	{
10231	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10232	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10233	}
10234
10235	if ( !fLongModeGuest
10236	&& (u32GuestCr4 & X86_CR4_PCIDE))
10237	{
10238	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10239	}
10240
10241	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10242	* 51:32 beyond the processor's physical-address width are 0. */
10243
10244	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10245	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10246	{
10247	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10248	}
10249
10250	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10251	AssertRCBreak(rc);
10252	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10253
10254	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10255	AssertRCBreak(rc);
10256	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10257	#endif
10258
10259	/*
10260	* PERF_GLOBAL MSR.
10261	*/
10262	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10263	{
10264	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10265	AssertRCBreak(rc);
10266	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10267	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10268	}
10269
10270	/*
10271	* PAT MSR.
10272	*/
10273	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10274	{
10275	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10276	AssertRCBreak(rc);
10277	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10278	for (unsigned i = 0; i < 8; i++)
10279	{
10280	uint8_t u8Val = (u64Val & 0xff);
10281	if ( u8Val != 0 /* UC */
10282	&& u8Val != 1 /* WC */
10283	&& u8Val != 4 /* WT */
10284	&& u8Val != 5 /* WP */
10285	&& u8Val != 6 /* WB */
10286	&& u8Val != 7 /* UC- */)
10287	{
10288	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10289	}
10290	u64Val >>= 8;
10291	}
10292	}
10293
10294	/*
10295	* EFER MSR.
10296	*/
10297	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10298	{
10299	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10300	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10301	AssertRCBreak(rc);
10302	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10303	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10304	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10305	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10306	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10307	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10308	* iemVmxVmentryCheckGuestState(). */
10309	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10310	\|\| !(u32GuestCr0 & X86_CR0_PG)
10311	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10312	VMX_IGS_EFER_LMA_LME_MISMATCH);
10313	}
10314
10315	/*
10316	* Segment registers.
10317	*/
10318	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10319	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10320	if (!(u32Eflags & X86_EFL_VM))
10321	{
10322	/* CS */
10323	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10324	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10325	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10326	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10327	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10328	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10329	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10330	/* CS cannot be loaded with NULL in protected mode. */
10331	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10332	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10333	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10334	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10335	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10336	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10337	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10338	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10339	else
10340	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10341
10342	/* SS */
10343	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10344	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10345	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10346	if ( !(pCtx->cr0 & X86_CR0_PE)
10347	\|\| pCtx->cs.Attr.n.u4Type == 3)
10348	{
10349	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10350	}
10351	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10352	{
10353	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10354	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10355	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10356	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10357	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10358	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10359	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10360	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10361	}
10362
10363	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10364	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10365	{
10366	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10367	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10368	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10369	\|\| pCtx->ds.Attr.n.u4Type > 11
10370	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10371	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10372	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10373	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10374	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10375	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10376	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10377	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10378	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10379	}
10380	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10381	{
10382	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10383	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10384	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10385	\|\| pCtx->es.Attr.n.u4Type > 11
10386	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10387	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10388	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10389	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10390	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10391	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10392	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10393	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10394	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10395	}
10396	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10397	{
10398	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10399	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10400	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10401	\|\| pCtx->fs.Attr.n.u4Type > 11
10402	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10403	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10404	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10405	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10406	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10407	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10408	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10409	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10410	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10411	}
10412	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10413	{
10414	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10415	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10416	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10417	\|\| pCtx->gs.Attr.n.u4Type > 11
10418	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10419	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10420	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10421	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10422	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10423	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10424	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10425	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10426	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10427	}
10428	/* 64-bit capable CPUs. */
10429	#if HC_ARCH_BITS == 64
10430	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10431	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10432	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10433	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10434	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10435	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10436	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10437	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10438	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10439	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10440	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10441	#endif
10442	}
10443	else
10444	{
10445	/* V86 mode checks. */
10446	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10447	if (pVmcsInfo->RealMode.fRealOnV86Active)
10448	{
10449	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10450	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10451	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10452	}
10453	else
10454	{
10455	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10456	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10457	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10458	}
10459
10460	/* CS */
10461	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10462	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10463	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10464	/* SS */
10465	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10466	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10467	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10468	/* DS */
10469	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10470	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10471	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10472	/* ES */
10473	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10474	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10475	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10476	/* FS */
10477	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10478	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10479	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10480	/* GS */
10481	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10482	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10483	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10484	/* 64-bit capable CPUs. */
10485	#if HC_ARCH_BITS == 64
10486	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10487	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10488	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10489	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10490	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10491	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10492	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10493	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10494	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10495	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10496	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10497	#endif
10498	}
10499
10500	/*
10501	* TR.
10502	*/
10503	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10504	/* 64-bit capable CPUs. */
10505	#if HC_ARCH_BITS == 64
10506	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10507	#endif
10508	if (fLongModeGuest)
10509	{
10510	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10511	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10512	}
10513	else
10514	{
10515	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10516	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10517	VMX_IGS_TR_ATTR_TYPE_INVALID);
10518	}
10519	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10520	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10521	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10522	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10523	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10524	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10525	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10526	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10527
10528	/*
10529	* GDTR and IDTR.
10530	*/
10531	#if HC_ARCH_BITS == 64
10532	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10533	AssertRCBreak(rc);
10534	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10535
10536	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10537	AssertRCBreak(rc);
10538	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10539	#endif
10540
10541	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10542	AssertRCBreak(rc);
10543	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10544
10545	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10546	AssertRCBreak(rc);
10547	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10548
10549	/*
10550	* Guest Non-Register State.
10551	*/
10552	/* Activity State. */
10553	uint32_t u32ActivityState;
10554	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10555	AssertRCBreak(rc);
10556	HMVMX_CHECK_BREAK( !u32ActivityState
10557	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10558	VMX_IGS_ACTIVITY_STATE_INVALID);
10559	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10560	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10561	uint32_t u32IntrState;
10562	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10563	AssertRCBreak(rc);
10564	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10565	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10566	{
10567	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10568	}
10569
10570	/** @todo Activity state and injecting interrupts. Left as a todo since we
10571	* currently don't use activity states but ACTIVE. */
10572
10573	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10574	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10575
10576	/* Guest interruptibility-state. */
10577	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10578	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10579	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10580	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10581	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10582	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10583	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10584	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10585	{
10586	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10587	{
10588	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10589	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10590	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10591	}
10592	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10593	{
10594	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10595	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10596	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10597	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10598	}
10599	}
10600	/** @todo Assumes the processor is not in SMM. */
10601	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10602	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10603	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10604	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10605	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10606	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10607	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10608	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10609	{
10610	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10611	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10612	}
10613
10614	/* Pending debug exceptions. */
10615	#if HC_ARCH_BITS == 64
10616	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10617	AssertRCBreak(rc);
10618	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10619	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10620	u32Val = u64Val; /* For pending debug exceptions checks below. */
10621	#else
10622	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10623	AssertRCBreak(rc);
10624	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10625	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10626	#endif
10627
10628	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10629	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10630	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10631	{
10632	if ( (u32Eflags & X86_EFL_TF)
10633	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10634	{
10635	/* Bit 14 is PendingDebug.BS. */
10636	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10637	}
10638	if ( !(u32Eflags & X86_EFL_TF)
10639	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10640	{
10641	/* Bit 14 is PendingDebug.BS. */
10642	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10643	}
10644	}
10645
10646	/* VMCS link pointer. */
10647	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10648	AssertRCBreak(rc);
10649	if (u64Val != UINT64_C(0xffffffffffffffff))
10650	{
10651	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10652	/** @todo Bits beyond the processor's physical-address width MBZ. */
10653	/** @todo SMM checks. */
10654	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10655	Assert(pVmcsInfo->pvShadowVmcs);
10656	VMXVMCSREVID VmcsRevId;
10657	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10658	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10659	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10660	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10661	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10662	}
10663
10664	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10665	* not using nested paging? */
10666	if ( pVM->hm.s.fNestedPaging
10667	&& !fLongModeGuest
10668	&& CPUMIsGuestInPAEModeEx(pCtx))
10669	{
10670	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10671	AssertRCBreak(rc);
10672	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10673
10674	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10675	AssertRCBreak(rc);
10676	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10677
10678	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10679	AssertRCBreak(rc);
10680	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10681
10682	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10683	AssertRCBreak(rc);
10684	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10685	}
10686
10687	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10688	if (uError == VMX_IGS_ERROR)
10689	uError = VMX_IGS_REASON_NOT_FOUND;
10690	} while (0);
10691
10692	pVCpu->hm.s.u32HMError = uError;
10693	return uError;
10694
10695	#undef HMVMX_ERROR_BREAK
10696	#undef HMVMX_CHECK_BREAK
10697	}
10698
10699
10700	/**
10701	* Map the APIC-access page for virtualizing APIC accesses.
10702	*
10703	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10704	* this not done as part of exporting guest state, see @bugref{8721}.
10705	*
10706	* @returns VBox status code.
10707	* @param pVCpu The cross context virtual CPU structure.
10708	*/
10709	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10710	{
10711	PVM pVM = pVCpu->CTX_SUFF(pVM);
10712	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10713
10714	Assert(PDMHasApic(pVM));
10715	Assert(u64MsrApicBase);
10716
10717	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10718	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10719
10720	/* Unalias the existing mapping. */
10721	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10722	AssertRCReturn(rc, rc);
10723
10724	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10725	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10726	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10727	AssertRCReturn(rc, rc);
10728
10729	/* Update the per-VCPU cache of the APIC base MSR. */
10730	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10731	return VINF_SUCCESS;
10732	}
10733
10734
10735	/**
10736	* Wrapper for dispatching host NMIs.
10737	*
10738	* @returns VBox status code.
10739	* @param pVCpu The cross context virtual CPU structure.
10740	*/
10741	static int hmR0VmxExitHostNmi(PVMCPU pVCpu)
10742	{
10743	VMXDispatchHostNmi();
10744	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10745	return VINF_SUCCESS;
10746	}
10747
10748
10749	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10750	/**
10751	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10752	* nested-guest using hardware-assisted VMX.
10753	*
10754	* @param pVCpu The cross context virtual CPU structure.
10755	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10756	* @param pVmcsInfoGst The guest VMCS info. object.
10757	*/
10758	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10759	{
10760	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10761	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10762	Assert(pu64MsrBitmap);
10763
10764	/*
10765	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10766	* MSR that is intercepted by the guest is also intercepted while executing the
10767	* nested-guest using hardware-assisted VMX.
10768	*
10769	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10770	* nested-guest VM-exit even if the outer guest is not intercepting some
10771	* MSRs. We cannot assume the caller has initialized the nested-guest
10772	* MSR bitmap in this case.
10773	*
10774	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10775	* each VM-entry, hence initializing it once per-VM while setting up the
10776	* nested-guest VMCS is not sufficient.
10777	*/
10778	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10779	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10780	{
10781	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10782	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10783	Assert(pu64MsrBitmapNstGst);
10784	Assert(pu64MsrBitmapGst);
10785
10786	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10787	for (uint32_t i = 0; i < cFrags; i++)
10788	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10789	}
10790	else
10791	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10792	}
10793
10794
10795	/**
10796	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10797	* hardware-assisted VMX execution of the nested-guest.
10798	*
10799	* For a guest, we don't modify these controls once we set up the VMCS and hence
10800	* this function is never called.
10801	*
10802	* For nested-guests since the guest hypervisor provides these controls on every
10803	* nested-guest VM-entry and could potentially change them everytime we need to
10804	* merge them before every nested-guest VM-entry.
10805	*
10806	* @returns VBox status code.
10807	* @param pVCpu The cross context virtual CPU structure.
10808	*/
10809	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10810	{
10811	PVM pVM = pVCpu->CTX_SUFF(pVM);
10812	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10813	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10814	Assert(pVmcsNstGst);
10815
10816	/*
10817	* Merge the controls with the requirements of the guest VMCS.
10818	*
10819	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10820	* VMCS with the features supported by the physical CPU as it's already done by the
10821	* VMLAUNCH/VMRESUME instruction emulation.
10822	*
10823	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10824	* derived from the VMX features supported by the physical CPU.
10825	*/
10826
10827	/* Pin-based VM-execution controls. */
10828	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10829
10830	/* Processor-based VM-execution controls. */
10831	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10832	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10833	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10834	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10835	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10836
10837	/* Secondary processor-based VM-execution controls. */
10838	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10839	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10840	\| VMX_PROC_CTLS2_INVPCID
10841	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10842	\| VMX_PROC_CTLS2_RDTSCP
10843	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10844	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10845	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10846	\| VMX_PROC_CTLS2_VMFUNC));
10847
10848	/*
10849	* VM-entry controls:
10850	* These controls contains state that depends on the nested-guest state (primarily
10851	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10852	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10853	* properly continue executing the nested-guest if the EFER MSR changes but does not
10854	* cause a nested-guest VM-exits.
10855	*
10856	* VM-exit controls:
10857	* These controls specify the host state on return. We cannot use the controls from
10858	* the guest hypervisor state as is as it would contain the guest state rather than
10859	* the host state. Since the host state is subject to change (e.g. preemption, trips
10860	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10861	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10862	*
10863	* VM-entry MSR-load:
10864	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10865	* context by the VMLAUNCH/VMRESUME instruction emulation.
10866	*
10867	* VM-exit MSR-store:
10868	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10869	* back into the VM-exit MSR-store area.
10870	*
10871	* VM-exit MSR-load areas:
10872	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10873	* can entirely ignore what the guest hypervisor wants to load here.
10874	*/
10875
10876	/*
10877	* Exception bitmap.
10878	*
10879	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10880	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10881	* code more flexible if intercepting exceptions become more dynamic in the future we do
10882	* it as part of exporting the nested-guest state.
10883	*/
10884	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10885
10886	/*
10887	* CR0/CR4 guest/host mask.
10888	*
10889	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10890	* cause VM-exits, so we need to merge them here.
10891	*/
10892	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10893	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10894
10895	/*
10896	* Page-fault error-code mask and match.
10897	*
10898	* Although we require unrestricted guest execution (and thereby nested-paging) for
10899	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10900	* normally intercept #PFs, it might intercept them for debugging purposes.
10901	*
10902	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10903	* If the outer guest is intercepting #PFs we must intercept all #PFs.
10904	*/
10905	uint32_t u32XcptPFMask;
10906	uint32_t u32XcptPFMatch;
10907	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10908	{
10909	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10910	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10911	}
10912	else
10913	{
10914	u32XcptPFMask = 0;
10915	u32XcptPFMatch = 0;
10916	}
10917
10918	/*
10919	* Pause-Loop exiting.
10920	*/
10921	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10922	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10923
10924	/*
10925	* I/O Bitmap.
10926	*
10927	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10928	* intercept all I/O port accesses.
10929	*/
10930	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10931
10932	/*
10933	* VMCS shadowing.
10934	*
10935	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10936	* enabled while executing the nested-guest.
10937	*/
10938	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10939
10940	/*
10941	* APIC-access page.
10942	*
10943	* The APIC-access page address has already been initialized while setting up the
10944	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it should
10945	* not be of any consequence to the host or to the guest for that matter, but we only
10946	* accept valid addresses verified by the VMLAUNCH/VMRESUME instruction emulation to keep
10947	* it simple.
10948	*/
10949
10950	/*
10951	* Virtual-APIC page and TPR threshold.
10952	*
10953	* The virtual-APIC page has already been allocated (by CPUM during VM startup) and cached
10954	* from guest memory as part of VMLAUNCH/VMRESUME instruction emulation. The host physical
10955	* address has also been updated in the nested-guest VMCS.
10956	*/
10957	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10958	RTHCPHYS HCPhysVirtApic;
10959	uint32_t u32TprThreshold;
10960	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10961	{
10962	HCPhysVirtApic = pVmcsInfoNstGst->HCPhysVirtApic;
10963	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10964	}
10965	else
10966	{
10967	HCPhysVirtApic = 0;
10968	u32TprThreshold = 0;
10969
10970	/*
10971	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10972	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10973	* be taken care of by EPT/shadow paging.
10974	*/
10975	if (pVM->hm.s.fAllow64BitGuests)
10976	{
10977	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10978	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10979	}
10980	}
10981
10982	/*
10983	* Validate basic assumptions.
10984	*/
10985	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10986	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10987	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10988
10989	/*
10990	* Commit it to the nested-guest VMCS.
10991	*/
10992	int rc = VINF_SUCCESS;
10993	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10994	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10995	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10996	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10997	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10998	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10999	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
11000	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
11001	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
11002	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
11003	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
11004	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
11005	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
11006	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
11007	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
11008	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
11009	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
11010	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
11011	{
11012	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
11013	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
11014	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
11015	}
11016	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
11017	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVmcsInfoNstGst->HCPhysVirtApic);
11018	AssertRCReturn(rc, rc);
11019
11020	/*
11021	* Update the nested-guest VMCS cache.
11022	*/
11023	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
11024	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
11025	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
11026	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
11027	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
11028	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
11029	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
11030	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
11031	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
11032
11033	/*
11034	* MSR bitmap.
11035	*
11036	* The MSR bitmap address has already been initialized while setting up the nested-guest
11037	* VMCS, here we need to merge the MSR bitmaps.
11038	*/
11039	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11040	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
11041
11042	return VINF_SUCCESS;
11043	}
11044	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11045
11046
11047	/**
11048	* Does the preparations before executing guest code in VT-x.
11049	*
11050	* This may cause longjmps to ring-3 and may even result in rescheduling to the
11051	* recompiler/IEM. We must be cautious what we do here regarding committing
11052	* guest-state information into the VMCS assuming we assuredly execute the
11053	* guest in VT-x mode.
11054	*
11055	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
11056	* the common-state (TRPM/forceflags), we must undo those changes so that the
11057	* recompiler/IEM can (and should) use them when it resumes guest execution.
11058	* Otherwise such operations must be done when we can no longer exit to ring-3.
11059	*
11060	* @returns Strict VBox status code (i.e. informational status codes too).
11061	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
11062	* have been disabled.
11063	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
11064	* pending events).
11065	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
11066	* double-fault into the guest.
11067	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
11068	* dispatched directly.
11069	* @retval VINF_* scheduling changes, we have to go back to ring-3.
11070	*
11071	* @param pVCpu The cross context virtual CPU structure.
11072	* @param pVmxTransient The VMX-transient structure.
11073	* @param fStepping Whether we are single-stepping the guest in the
11074	* hypervisor debugger. Makes us ignore some of the reasons
11075	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
11076	* if event dispatching took place.
11077	*/
11078	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
11079	{
11080	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11081
11082	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
11083	if (pVmxTransient->fIsNestedGuest)
11084	{
11085	RT_NOREF2(pVCpu, fStepping);
11086	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
11087	return VINF_EM_RESCHEDULE_REM;
11088	}
11089	#endif
11090
11091	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
11092	PGMRZDynMapFlushAutoSet(pVCpu);
11093	#endif
11094
11095	/*
11096	* Check and process force flag actions, some of which might require us to go back to ring-3.
11097	*/
11098	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11099	if (rcStrict == VINF_SUCCESS)
11100	{ /* FFs don't get set all the time. */ }
11101	else
11102	return rcStrict;
11103
11104	/*
11105	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
11106	*/
11107	/** @todo Doing this from ring-3 after VM setup phase causes a
11108	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
11109	* idea why atm. */
11110	PVM pVM = pVCpu->CTX_SUFF(pVM);
11111	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11112	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11113	&& PDMHasApic(pVM))
11114	{
11115	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11116	AssertRCReturn(rc, rc);
11117	}
11118
11119	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11120	/*
11121	* Merge guest VMCS controls with the nested-guest VMCS controls.
11122	*
11123	* Even if we have not executed the guest prior to this (e.g. when resuming from a
11124	* saved state), we should be okay with merging controls as we initialize the
11125	* guest VMCS controls as part of VM setup phase.
11126	*/
11127	if ( pVmxTransient->fIsNestedGuest
11128	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
11129	{
11130	int rc = hmR0VmxMergeVmcsNested(pVCpu);
11131	AssertRCReturn(rc, rc);
11132	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11133	}
11134	#endif
11135
11136	/*
11137	* Evaluate events to be injected into the guest.
11138	*
11139	* Events in TRPM can be injected without inspecting the guest state.
11140	* If any new events (interrupts/NMI) are pending currently, we try to set up the
11141	* guest to cause a VM-exit the next time they are ready to receive the event.
11142	*
11143	* With nested-guests, evaluating pending events may cause VM-exits.
11144	*/
11145	if (TRPMHasTrap(pVCpu))
11146	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11147
11148	uint32_t fIntrState;
11149	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11150
11151	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11152	/*
11153	* While evaluating pending events if something failed (unlikely) or if we were
11154	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11155	*/
11156	if (rcStrict != VINF_SUCCESS)
11157	return rcStrict;
11158	if ( pVmxTransient->fIsNestedGuest
11159	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11160	{
11161	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11162	return VINF_VMX_VMEXIT;
11163	}
11164	#else
11165	Assert(rcStrict == VINF_SUCCESS);
11166	#endif
11167
11168	/*
11169	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11170	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11171	* also result in triple-faulting the VM.
11172	*
11173	* With nested-guests, the above does not apply since unrestricted guest execution is a
11174	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
11175	*/
11176	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11177	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11178	{ /* likely */ }
11179	else
11180	{
11181	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11182	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11183	return rcStrict;
11184	}
11185
11186	/*
11187	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
11188	* import CR3 themselves. We will need to update them here, as even as late as the above
11189	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11190	* the below force flags to be set.
11191	*/
11192	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11193	{
11194	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11195	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11196	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
11197	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11198	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11199	}
11200	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11201	{
11202	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11203	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11204	}
11205
11206	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11207	/* Paranoia. */
11208	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11209	#endif
11210
11211	/*
11212	* No longjmps to ring-3 from this point on!!!
11213	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11214	* This also disables flushing of the R0-logger instance (if any).
11215	*/
11216	VMMRZCallRing3Disable(pVCpu);
11217
11218	/*
11219	* Export the guest state bits.
11220	*
11221	* We cannot perform longjmps while loading the guest state because we do not preserve the
11222	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
11223	* CPU migration.
11224	*
11225	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11226	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
11227	*/
11228	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11229	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11230	{ /* likely */ }
11231	else
11232	{
11233	VMMRZCallRing3Enable(pVCpu);
11234	return rcStrict;
11235	}
11236
11237	/*
11238	* We disable interrupts so that we don't miss any interrupts that would flag preemption
11239	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11240	* preemption disabled for a while. Since this is purely to aid the
11241	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11242	* disable interrupt on NT.
11243	*
11244	* We need to check for force-flags that could've possible been altered since we last
11245	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11246	* see @bugref{6398}).
11247	*
11248	* We also check a couple of other force-flags as a last opportunity to get the EMT back
11249	* to ring-3 before executing guest code.
11250	*/
11251	pVmxTransient->fEFlags = ASMIntDisableFlags();
11252
11253	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
11254	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11255	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11256	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
11257	{
11258	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11259	{
11260	pVCpu->hm.s.Event.fPending = false;
11261
11262	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11263	/*
11264	* If we are executing a nested-guest make sure that we should intercept subsequent
11265	* events. The one we are injecting might be part of VM-entry.
11266	*/
11267	if (pVmxTransient->fIsNestedGuest)
11268	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
11269	#endif
11270
11271	/*
11272	* We've injected any pending events. This is really the point of no return (to ring-3).
11273	*
11274	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
11275	* returns from this function, so don't enable them here.
11276	*/
11277	return VINF_SUCCESS;
11278	}
11279
11280	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11281	rcStrict = VINF_EM_RAW_INTERRUPT;
11282	}
11283	else
11284	{
11285	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11286	rcStrict = VINF_EM_RAW_TO_R3;
11287	}
11288
11289	ASMSetFlags(pVmxTransient->fEFlags);
11290	VMMRZCallRing3Enable(pVCpu);
11291
11292	return rcStrict;
11293	}
11294
11295
11296	/**
11297	* Final preparations before executing guest code using hardware-assisted VMX.
11298	*
11299	* We can no longer get preempted to a different host CPU and there are no returns
11300	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11301	* failures), this function is not intended to fail sans unrecoverable hardware
11302	* errors.
11303	*
11304	* @param pVCpu The cross context virtual CPU structure.
11305	* @param pVmxTransient The VMX-transient structure.
11306	*
11307	* @remarks Called with preemption disabled.
11308	* @remarks No-long-jump zone!!!
11309	*/
11310	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11311	{
11312	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11313	Assert(VMMR0IsLogFlushDisabled(pVCpu));
11314	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11315	Assert(!pVCpu->hm.s.Event.fPending);
11316
11317	/*
11318	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11319	*/
11320	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11321	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11322
11323	PVM pVM = pVCpu->CTX_SUFF(pVM);
11324	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11325	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11326	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11327
11328	if (!CPUMIsGuestFPUStateActive(pVCpu))
11329	{
11330	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11331	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11332	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
11333	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11334	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11335	}
11336
11337	/*
11338	* Re-export the host state bits as we may've been preempted (only happens when
11339	* thread-context hooks are used or when the VM start function changes) or if
11340	* the host CR0 is modified while loading the guest FPU state above.
11341	*
11342	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11343	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
11344	* see @bugref{8432}.
11345	*
11346	* This may also happen when switching to/from a nested-guest VMCS without leaving
11347	* ring-0.
11348	*/
11349	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11350	{
11351	hmR0VmxExportHostState(pVCpu);
11352	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
11353	}
11354	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11355
11356	/*
11357	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
11358	*/
11359	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11360	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11361	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11362
11363	/*
11364	* Store status of the shared guest/host debug state at the time of VM-entry.
11365	*/
11366	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11367	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11368	{
11369	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11370	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11371	}
11372	else
11373	#endif
11374	{
11375	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11376	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11377	}
11378
11379	/*
11380	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11381	* more than one conditional check. The post-run side of our code shall determine
11382	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11383	*/
11384	if (pVmcsInfo->pbVirtApic)
11385	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11386
11387	/*
11388	* Update the host MSRs values in the VM-exit MSR-load area.
11389	*/
11390	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11391	{
11392	if (pVmcsInfo->cExitMsrLoad > 0)
11393	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11394	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11395	}
11396
11397	/*
11398	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11399	* VMX-preemption timer based on the next virtual sync clock deadline.
11400	*/
11401	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11402	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
11403	{
11404	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11405	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11406	}
11407
11408	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11409	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11410	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11411	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11412	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
11413
11414	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11415
11416	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11417	as we're about to start executing the guest . */
11418
11419	/*
11420	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11421	*
11422	* This is done this late as updating the TSC offsetting/preemption timer above
11423	* figures out if we can skip intercepting RDTSCP by calculating the number of
11424	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
11425	*/
11426	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11427	&& !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11428	{
11429	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11430
11431	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11432	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11433	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11434	true /* fSetReadWrite /, true / fUpdateHostMsr */);
11435	AssertRC(rc);
11436	Assert(!pVmxTransient->fRemoveTscAuxMsr);
11437	pVmxTransient->fRemoveTscAuxMsr = true;
11438	}
11439
11440	#ifdef VBOX_STRICT
11441	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11442	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11443	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11444	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
11445	#endif
11446
11447	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11448	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11449	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11450	* see @bugref{9180#c54}. */
11451	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11452	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11453	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11454	#endif
11455	}
11456
11457
11458	/**
11459	* First C routine invoked after running guest code using hardware-assisted VMX.
11460	*
11461	* @param pVCpu The cross context virtual CPU structure.
11462	* @param pVmxTransient The VMX-transient structure.
11463	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11464	*
11465	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
11466	*
11467	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
11468	* unconditionally when it is safe to do so.
11469	*/
11470	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11471	{
11472	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11473
11474	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11475	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11476	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11477	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11478	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11479	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11480
11481	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11482	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11483	{
11484	uint64_t uGstTsc;
11485	if (!pVmxTransient->fIsNestedGuest)
11486	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11487	else
11488	{
11489	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11490	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11491	}
11492	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11493	}
11494
11495	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11496	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11497	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11498
11499	#if HC_ARCH_BITS == 64
11500	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11501	#endif
11502	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11503	/* The 64-on-32 switcher maintains VMCS-launch state on its own
11504	and we need to leave it alone here. */
11505	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11506	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11507	#else
11508	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11509	#endif
11510	#ifdef VBOX_STRICT
11511	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11512	#endif
11513	Assert(!ASMIntAreEnabled());
11514	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11515	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11516
11517	/*
11518	* Save the basic VM-exit reason and check if the VM-entry failed.
11519	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11520	*/
11521	uint32_t uExitReason;
11522	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11523	AssertRC(rc);
11524	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11525	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11526
11527	/*
11528	* Log the VM-exit before logging anything else as otherwise it might be a
11529	* tad confusing what happens before and after the world-switch.
11530	*/
11531	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11532
11533	/*
11534	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11535	* bitmap permissions, if it was added before VM-entry.
11536	*/
11537	if (pVmxTransient->fRemoveTscAuxMsr)
11538	{
11539	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11540	pVmxTransient->fRemoveTscAuxMsr = false;
11541	}
11542
11543	/*
11544	* Check if VMLAUNCH/VMRESUME succeeded.
11545	* If this failed, we cause a guru meditation and cease further execution.
11546	*
11547	* However, if we are executing a nested-guest we might fail if we use the
11548	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11549	*/
11550	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11551	{
11552	/*
11553	* Update the VM-exit history array here even if the VM-entry failed due to:
11554	* - Invalid guest state.
11555	* - MSR loading.
11556	* - Machine-check event.
11557	*
11558	* In any of the above cases we will still have a "valid" VM-exit reason
11559	* despite @a fVMEntryFailed being false.
11560	*
11561	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11562	*
11563	* Note! We don't have CS or RIP at this point. Will probably address that later
11564	* by amending the history entry added here.
11565	*/
11566	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11567	UINT64_MAX, uHostTsc);
11568
11569	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11570	{
11571	VMMRZCallRing3Enable(pVCpu);
11572
11573	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11574	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11575
11576	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11577	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11578	AssertRC(rc);
11579	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11580	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11581	AssertRC(rc);
11582	#else
11583	/*
11584	* Import the guest-interruptibility state always as we need it while evaluating
11585	* injecting events on re-entry.
11586	*
11587	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11588	* checking for real-mode while exporting the state because all bits that cause
11589	* mode changes wrt CR0 are intercepted.
11590	*/
11591	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11592	AssertRC(rc);
11593	#endif
11594
11595	/*
11596	* Sync the TPR shadow with our APIC state.
11597	*
11598	* With nested-guests, mark the virtual-APIC page as dirty so it can be synced
11599	* when performing the nested-guest VM-exit.
11600	*/
11601	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11602	{
11603	if (!pVmxTransient->fIsNestedGuest)
11604	{
11605	Assert(pVmcsInfo->pbVirtApic);
11606	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11607	{
11608	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11609	AssertRC(rc);
11610	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11611	}
11612	}
11613	else
11614	pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtApicPageDirty = true;
11615	}
11616
11617	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11618	return;
11619	}
11620	}
11621	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11622	else if (pVmxTransient->fIsNestedGuest)
11623	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11624	#endif
11625	else
11626	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11627
11628	VMMRZCallRing3Enable(pVCpu);
11629	}
11630
11631
11632	/**
11633	* Runs the guest code using hardware-assisted VMX the normal way.
11634	*
11635	* @returns VBox status code.
11636	* @param pVCpu The cross context virtual CPU structure.
11637	* @param pcLoops Pointer to the number of executed loops.
11638	*/
11639	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11640	{
11641	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11642	Assert(pcLoops);
11643	Assert(*pcLoops <= cMaxResumeLoops);
11644	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11645
11646	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11647	/*
11648	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11649	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11650	* guest VMCS while entering the VMX ring-0 session.
11651	*/
11652	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11653	{
11654	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11655	if (RT_SUCCESS(rc))
11656	{ /* likely */ }
11657	else
11658	{
11659	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11660	return rc;
11661	}
11662	}
11663	#endif
11664
11665	VMXTRANSIENT VmxTransient;
11666	RT_ZERO(VmxTransient);
11667	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11668
11669	/* Paranoia. */
11670	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11671
11672	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11673	for (;;)
11674	{
11675	Assert(!HMR0SuspendPending());
11676	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11677	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11678
11679	/*
11680	* Preparatory work for running nested-guest code, this may force us to
11681	* return to ring-3.
11682	*
11683	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11684	*/
11685	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11686	if (rcStrict != VINF_SUCCESS)
11687	break;
11688
11689	/* Interrupts are disabled at this point! */
11690	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11691	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11692	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11693	/* Interrupts are re-enabled at this point! */
11694
11695	/*
11696	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11697	*/
11698	if (RT_SUCCESS(rcRun))
11699	{ /* very likely */ }
11700	else
11701	{
11702	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11703	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11704	return rcRun;
11705	}
11706
11707	/*
11708	* Profile the VM-exit.
11709	*/
11710	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11712	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11713	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11714	HMVMX_START_EXIT_DISPATCH_PROF();
11715
11716	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11717
11718	/*
11719	* Handle the VM-exit.
11720	*/
11721	#ifdef HMVMX_USE_FUNCTION_TABLE
11722	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11723	#else
11724	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11725	#endif
11726	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11727	if (rcStrict == VINF_SUCCESS)
11728	{
11729	if (++(*pcLoops) <= cMaxResumeLoops)
11730	continue;
11731	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11732	rcStrict = VINF_EM_RAW_INTERRUPT;
11733	}
11734	break;
11735	}
11736
11737	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11738	return rcStrict;
11739	}
11740
11741
11742	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11743	/**
11744	* Runs the nested-guest code using hardware-assisted VMX.
11745	*
11746	* @returns VBox status code.
11747	* @param pVCpu The cross context virtual CPU structure.
11748	* @param pcLoops Pointer to the number of executed loops.
11749	*
11750	* @sa hmR0VmxRunGuestCodeNormal.
11751	*/
11752	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11753	{
11754	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11755	Assert(pcLoops);
11756	Assert(*pcLoops <= cMaxResumeLoops);
11757	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11758
11759	/*
11760	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11761	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11762	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11763	*/
11764	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11765	{
11766	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11767	if (RT_SUCCESS(rc))
11768	{ /* likely */ }
11769	else
11770	{
11771	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11772	return rc;
11773	}
11774	}
11775
11776	VMXTRANSIENT VmxTransient;
11777	RT_ZERO(VmxTransient);
11778	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11779	VmxTransient.fIsNestedGuest = true;
11780
11781	/* Paranoia. */
11782	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11783
11784	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11785	for (;;)
11786	{
11787	Assert(!HMR0SuspendPending());
11788	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11789	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11790
11791	/*
11792	* Preparatory work for running guest code, this may force us to
11793	* return to ring-3.
11794	*
11795	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11796	*/
11797	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11798	if (rcStrict != VINF_SUCCESS)
11799	break;
11800
11801	/* Interrupts are disabled at this point! */
11802	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11803	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11804	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11805	/* Interrupts are re-enabled at this point! */
11806
11807	/*
11808	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11809	*/
11810	if (RT_SUCCESS(rcRun))
11811	{ /* very likely */ }
11812	else
11813	{
11814	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11815	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11816	return rcRun;
11817	}
11818
11819	/*
11820	* Profile the VM-exit.
11821	*/
11822	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11824	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11825	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11826	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11827	HMVMX_START_EXIT_DISPATCH_PROF();
11828
11829	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11830
11831	/*
11832	* Handle the VM-exit.
11833	*/
11834	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11835	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11836	if (rcStrict == VINF_SUCCESS)
11837	{
11838	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11839	{
11840	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11841	rcStrict = VINF_VMX_VMEXIT;
11842	}
11843	else
11844	{
11845	if (++(*pcLoops) <= cMaxResumeLoops)
11846	continue;
11847	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11848	rcStrict = VINF_EM_RAW_INTERRUPT;
11849	}
11850	}
11851	else
11852	Assert(rcStrict != VINF_VMX_VMEXIT);
11853	break;
11854	}
11855
11856	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11857	return rcStrict;
11858	}
11859	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11860
11861
11862	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11863	* probes.
11864	*
11865	* The following few functions and associated structure contains the bloat
11866	* necessary for providing detailed debug events and dtrace probes as well as
11867	* reliable host side single stepping. This works on the principle of
11868	* "subclassing" the normal execution loop and workers. We replace the loop
11869	* method completely and override selected helpers to add necessary adjustments
11870	* to their core operation.
11871	*
11872	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11873	* any performance for debug and analysis features.
11874	*
11875	* @{
11876	*/
11877
11878	/**
11879	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11880	* the debug run loop.
11881	*/
11882	typedef struct VMXRUNDBGSTATE
11883	{
11884	/** The RIP we started executing at. This is for detecting that we stepped. */
11885	uint64_t uRipStart;
11886	/** The CS we started executing with. */
11887	uint16_t uCsStart;
11888
11889	/** Whether we've actually modified the 1st execution control field. */
11890	bool fModifiedProcCtls : 1;
11891	/** Whether we've actually modified the 2nd execution control field. */
11892	bool fModifiedProcCtls2 : 1;
11893	/** Whether we've actually modified the exception bitmap. */
11894	bool fModifiedXcptBitmap : 1;
11895
11896	/** We desire the modified the CR0 mask to be cleared. */
11897	bool fClearCr0Mask : 1;
11898	/** We desire the modified the CR4 mask to be cleared. */
11899	bool fClearCr4Mask : 1;
11900	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11901	uint32_t fCpe1Extra;
11902	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11903	uint32_t fCpe1Unwanted;
11904	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11905	uint32_t fCpe2Extra;
11906	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11907	uint32_t bmXcptExtra;
11908	/** The sequence number of the Dtrace provider settings the state was
11909	* configured against. */
11910	uint32_t uDtraceSettingsSeqNo;
11911	/** VM-exits to check (one bit per VM-exit). */
11912	uint32_t bmExitsToCheck[3];
11913
11914	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11915	uint32_t fProcCtlsInitial;
11916	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11917	uint32_t fProcCtls2Initial;
11918	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11919	uint32_t bmXcptInitial;
11920	} VMXRUNDBGSTATE;
11921	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11922	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11923
11924
11925	/**
11926	* Initializes the VMXRUNDBGSTATE structure.
11927	*
11928	* @param pVCpu The cross context virtual CPU structure of the
11929	* calling EMT.
11930	* @param pVmxTransient The VMX-transient structure.
11931	* @param pDbgState The debug state to initialize.
11932	*/
11933	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11934	{
11935	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11936	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11937
11938	pDbgState->fModifiedProcCtls = false;
11939	pDbgState->fModifiedProcCtls2 = false;
11940	pDbgState->fModifiedXcptBitmap = false;
11941	pDbgState->fClearCr0Mask = false;
11942	pDbgState->fClearCr4Mask = false;
11943	pDbgState->fCpe1Extra = 0;
11944	pDbgState->fCpe1Unwanted = 0;
11945	pDbgState->fCpe2Extra = 0;
11946	pDbgState->bmXcptExtra = 0;
11947	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11948	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11949	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11950	}
11951
11952
11953	/**
11954	* Updates the VMSC fields with changes requested by @a pDbgState.
11955	*
11956	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11957	* immediately before executing guest code, i.e. when interrupts are disabled.
11958	* We don't check status codes here as we cannot easily assert or return in the
11959	* latter case.
11960	*
11961	* @param pVCpu The cross context virtual CPU structure.
11962	* @param pVmxTransient The VMX-transient structure.
11963	* @param pDbgState The debug state.
11964	*/
11965	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11966	{
11967	/*
11968	* Ensure desired flags in VMCS control fields are set.
11969	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11970	*
11971	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11972	* there should be no stale data in pCtx at this point.
11973	*/
11974	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11975	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11976	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11977	{
11978	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11979	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11980	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11981	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11982	pDbgState->fModifiedProcCtls = true;
11983	}
11984
11985	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11986	{
11987	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11988	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11989	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11990	pDbgState->fModifiedProcCtls2 = true;
11991	}
11992
11993	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11994	{
11995	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11996	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11997	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11998	pDbgState->fModifiedXcptBitmap = true;
11999	}
12000
12001	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
12002	{
12003	pVmcsInfo->u64Cr0Mask = 0;
12004	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
12005	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
12006	}
12007
12008	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
12009	{
12010	pVmcsInfo->u64Cr4Mask = 0;
12011	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
12012	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
12013	}
12014
12015	NOREF(pVCpu);
12016	}
12017
12018
12019	/**
12020	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
12021	* re-entry next time around.
12022	*
12023	* @returns Strict VBox status code (i.e. informational status codes too).
12024	* @param pVCpu The cross context virtual CPU structure.
12025	* @param pVmxTransient The VMX-transient structure.
12026	* @param pDbgState The debug state.
12027	* @param rcStrict The return code from executing the guest using single
12028	* stepping.
12029	*/
12030	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
12031	VBOXSTRICTRC rcStrict)
12032	{
12033	/*
12034	* Restore VM-exit control settings as we may not reenter this function the
12035	* next time around.
12036	*/
12037	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12038
12039	/* We reload the initial value, trigger what we can of recalculations the
12040	next time around. From the looks of things, that's all that's required atm. */
12041	if (pDbgState->fModifiedProcCtls)
12042	{
12043	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
12044	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
12045	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
12046	AssertRCReturn(rc2, rc2);
12047	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
12048	}
12049
12050	/* We're currently the only ones messing with this one, so just restore the
12051	cached value and reload the field. */
12052	if ( pDbgState->fModifiedProcCtls2
12053	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
12054	{
12055	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
12056	AssertRCReturn(rc2, rc2);
12057	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
12058	}
12059
12060	/* If we've modified the exception bitmap, we restore it and trigger
12061	reloading and partial recalculation the next time around. */
12062	if (pDbgState->fModifiedXcptBitmap)
12063	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
12064
12065	return rcStrict;
12066	}
12067
12068
12069	/**
12070	* Configures VM-exit controls for current DBGF and DTrace settings.
12071	*
12072	* This updates @a pDbgState and the VMCS execution control fields to reflect
12073	* the necessary VM-exits demanded by DBGF and DTrace.
12074	*
12075	* @param pVCpu The cross context virtual CPU structure.
12076	* @param pVmxTransient The VMX-transient structure. May update
12077	* fUpdatedTscOffsettingAndPreemptTimer.
12078	* @param pDbgState The debug state.
12079	*/
12080	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12081	{
12082	/*
12083	* Take down the dtrace serial number so we can spot changes.
12084	*/
12085	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
12086	ASMCompilerBarrier();
12087
12088	/*
12089	* We'll rebuild most of the middle block of data members (holding the
12090	* current settings) as we go along here, so start by clearing it all.
12091	*/
12092	pDbgState->bmXcptExtra = 0;
12093	pDbgState->fCpe1Extra = 0;
12094	pDbgState->fCpe1Unwanted = 0;
12095	pDbgState->fCpe2Extra = 0;
12096	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
12097	pDbgState->bmExitsToCheck[i] = 0;
12098
12099	/*
12100	* Software interrupts (INT XXh) - no idea how to trigger these...
12101	*/
12102	PVM pVM = pVCpu->CTX_SUFF(pVM);
12103	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
12104	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
12105	{
12106	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12107	}
12108
12109	/*
12110	* INT3 breakpoints - triggered by #BP exceptions.
12111	*/
12112	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
12113	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12114
12115	/*
12116	* Exception bitmap and XCPT events+probes.
12117	*/
12118	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
12119	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
12120	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
12121
12122	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
12123	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
12124	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12125	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
12126	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
12127	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
12128	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
12129	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
12130	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
12131	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
12132	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
12133	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
12134	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
12135	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
12136	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
12137	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
12138	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
12139	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
12140
12141	if (pDbgState->bmXcptExtra)
12142	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12143
12144	/*
12145	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12146	*
12147	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12148	* So, when adding/changing/removing please don't forget to update it.
12149	*
12150	* Some of the macros are picking up local variables to save horizontal space,
12151	* (being able to see it in a table is the lesser evil here).
12152	*/
12153	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12154	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12155	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12156	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12157	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12158	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12159	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12160	} else do { } while (0)
12161	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12162	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12163	{ \
12164	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
12165	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12166	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12167	} else do { } while (0)
12168	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12169	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12170	{ \
12171	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
12172	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12173	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12174	} else do { } while (0)
12175	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12176	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12177	{ \
12178	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
12179	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12180	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12181	} else do { } while (0)
12182
12183	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12184	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12185	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12186	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12187	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12188
12189	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12190	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12191	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12192	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12193	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12194	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12195	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12196	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12197	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12198	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12199	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12200	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12201	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12202	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12203	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12204	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12205	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12206	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12207	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12208	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12209	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12210	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12211	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12212	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12213	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12214	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12215	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12216	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12217	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12218	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12219	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12220	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12221	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12222	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12223	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12224	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12225
12226	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12227	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12228	{
12229	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
12230	\| CPUMCTX_EXTRN_APIC_TPR);
12231	AssertRC(rc);
12232
12233	#if 0 /** @todo fix me */
12234	pDbgState->fClearCr0Mask = true;
12235	pDbgState->fClearCr4Mask = true;
12236	#endif
12237	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12238	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
12239	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12240	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
12241	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12242	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12243	require clearing here and in the loop if we start using it. */
12244	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12245	}
12246	else
12247	{
12248	if (pDbgState->fClearCr0Mask)
12249	{
12250	pDbgState->fClearCr0Mask = false;
12251	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12252	}
12253	if (pDbgState->fClearCr4Mask)
12254	{
12255	pDbgState->fClearCr4Mask = false;
12256	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12257	}
12258	}
12259	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12260	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12261
12262	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12263	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12264	{
12265	/** @todo later, need to fix handler as it assumes this won't usually happen. */
12266	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12267	}
12268	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12269	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12270
12271	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12272	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12273	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12274	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12275	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12276	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12277	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12278	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12279	#if 0 /** @todo too slow, fix handler. */
12280	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12281	#endif
12282	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12283
12284	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12285	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12286	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12287	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12288	{
12289	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12290	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12291	}
12292	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12293	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12294	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12295	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12296
12297	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12298	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
12299	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12300	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
12301	{
12302	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12303	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12304	}
12305	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12306	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12307	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12308	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12309
12310	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12311	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12312	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12313	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12314	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12315	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12316	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12317	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12318	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12319	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12320	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12321	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12322	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12323	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12324	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12325	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12326	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12327	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12328	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12329	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12330	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12331	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12332
12333	#undef IS_EITHER_ENABLED
12334	#undef SET_ONLY_XBM_IF_EITHER_EN
12335	#undef SET_CPE1_XBM_IF_EITHER_EN
12336	#undef SET_CPEU_XBM_IF_EITHER_EN
12337	#undef SET_CPE2_XBM_IF_EITHER_EN
12338
12339	/*
12340	* Sanitize the control stuff.
12341	*/
12342	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12343	if (pDbgState->fCpe2Extra)
12344	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12345	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12346	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12347	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12348	{
12349	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12350	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12351	}
12352
12353	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12354	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12355	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12356	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12357	}
12358
12359
12360	/**
12361	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
12362	* appropriate.
12363	*
12364	* The caller has checked the VM-exit against the
12365	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12366	* already, so we don't have to do that either.
12367	*
12368	* @returns Strict VBox status code (i.e. informational status codes too).
12369	* @param pVCpu The cross context virtual CPU structure.
12370	* @param pVmxTransient The VMX-transient structure.
12371	* @param uExitReason The VM-exit reason.
12372	*
12373	* @remarks The name of this function is displayed by dtrace, so keep it short
12374	* and to the point. No longer than 33 chars long, please.
12375	*/
12376	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12377	{
12378	/*
12379	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12380	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
12381	*
12382	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12383	* does. Must add/change/remove both places. Same ordering, please.
12384	*
12385	* Added/removed events must also be reflected in the next section
12386	* where we dispatch dtrace events.
12387	*/
12388	bool fDtrace1 = false;
12389	bool fDtrace2 = false;
12390	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12391	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12392	uint32_t uEventArg = 0;
12393	#define SET_EXIT(a_EventSubName) \
12394	do { \
12395	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12396	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12397	} while (0)
12398	#define SET_BOTH(a_EventSubName) \
12399	do { \
12400	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12401	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12402	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12403	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12404	} while (0)
12405	switch (uExitReason)
12406	{
12407	case VMX_EXIT_MTF:
12408	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12409
12410	case VMX_EXIT_XCPT_OR_NMI:
12411	{
12412	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12413	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12414	{
12415	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12416	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12417	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12418	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12419	{
12420	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12421	{
12422	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12423	uEventArg = pVmxTransient->uExitIntErrorCode;
12424	}
12425	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12426	switch (enmEvent1)
12427	{
12428	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12429	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12430	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12431	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12432	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12433	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12434	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12435	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12436	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12437	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12438	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12439	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12440	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12441	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12442	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12443	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12444	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12445	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12446	default: break;
12447	}
12448	}
12449	else
12450	AssertFailed();
12451	break;
12452
12453	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12454	uEventArg = idxVector;
12455	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12456	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12457	break;
12458	}
12459	break;
12460	}
12461
12462	case VMX_EXIT_TRIPLE_FAULT:
12463	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12464	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12465	break;
12466	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12467	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12468	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12469	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12470	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12471
12472	/* Instruction specific VM-exits: */
12473	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12474	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12475	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12476	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12477	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12478	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12479	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12480	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12481	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12482	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12483	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12484	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12485	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12486	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12487	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12488	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12489	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12490	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12491	case VMX_EXIT_MOV_CRX:
12492	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12493	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12494	SET_BOTH(CRX_READ);
12495	else
12496	SET_BOTH(CRX_WRITE);
12497	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12498	break;
12499	case VMX_EXIT_MOV_DRX:
12500	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12501	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12502	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12503	SET_BOTH(DRX_READ);
12504	else
12505	SET_BOTH(DRX_WRITE);
12506	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12507	break;
12508	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12509	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12510	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12511	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12512	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12513	case VMX_EXIT_GDTR_IDTR_ACCESS:
12514	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12515	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12516	{
12517	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12518	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12519	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12520	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12521	}
12522	break;
12523
12524	case VMX_EXIT_LDTR_TR_ACCESS:
12525	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12526	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12527	{
12528	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12529	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12530	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12531	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12532	}
12533	break;
12534
12535	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12536	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12537	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12538	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12539	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12540	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12541	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12542	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12543	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12544	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12545	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12546
12547	/* Events that aren't relevant at this point. */
12548	case VMX_EXIT_EXT_INT:
12549	case VMX_EXIT_INT_WINDOW:
12550	case VMX_EXIT_NMI_WINDOW:
12551	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12552	case VMX_EXIT_PREEMPT_TIMER:
12553	case VMX_EXIT_IO_INSTR:
12554	break;
12555
12556	/* Errors and unexpected events. */
12557	case VMX_EXIT_INIT_SIGNAL:
12558	case VMX_EXIT_SIPI:
12559	case VMX_EXIT_IO_SMI:
12560	case VMX_EXIT_SMI:
12561	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12562	case VMX_EXIT_ERR_MSR_LOAD:
12563	case VMX_EXIT_ERR_MACHINE_CHECK:
12564	case VMX_EXIT_PML_FULL:
12565	case VMX_EXIT_VIRTUALIZED_EOI:
12566	break;
12567
12568	default:
12569	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12570	break;
12571	}
12572	#undef SET_BOTH
12573	#undef SET_EXIT
12574
12575	/*
12576	* Dtrace tracepoints go first. We do them here at once so we don't
12577	* have to copy the guest state saving and stuff a few dozen times.
12578	* Down side is that we've got to repeat the switch, though this time
12579	* we use enmEvent since the probes are a subset of what DBGF does.
12580	*/
12581	if (fDtrace1 \|\| fDtrace2)
12582	{
12583	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12584	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12585	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12586	switch (enmEvent1)
12587	{
12588	/** @todo consider which extra parameters would be helpful for each probe. */
12589	case DBGFEVENT_END: break;
12590	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12591	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12592	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12593	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12594	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12595	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12596	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12597	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12598	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12599	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12600	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12601	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12602	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12603	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12604	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12605	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12606	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12607	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12608	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12609	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12610	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12611	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12612	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12613	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12614	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12615	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12616	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12617	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12618	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12619	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12620	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12621	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12622	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12623	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12624	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12625	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12626	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12627	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12628	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12629	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12630	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12631	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12632	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12633	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12634	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12635	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12636	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12637	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12638	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12639	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12640	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12641	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12642	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12643	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12644	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12645	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12646	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12647	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12648	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12649	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12650	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12651	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12652	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12653	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12654	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12655	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12656	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12657	}
12658	switch (enmEvent2)
12659	{
12660	/** @todo consider which extra parameters would be helpful for each probe. */
12661	case DBGFEVENT_END: break;
12662	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12663	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12664	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12665	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12666	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12667	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12668	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12669	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12670	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12671	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12672	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12673	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12674	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12675	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12676	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12677	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12678	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12679	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12680	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12681	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12682	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12683	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12684	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12685	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12686	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12687	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12688	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12689	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12690	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12691	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12692	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12693	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12694	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12695	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12696	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12697	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12698	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12699	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12700	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12701	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12702	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12703	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12704	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12705	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12706	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12707	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12708	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12709	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12710	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12711	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12712	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12713	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12714	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12715	}
12716	}
12717
12718	/*
12719	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12720	* the DBGF call will do a full check).
12721	*
12722	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12723	* Note! If we have to events, we prioritize the first, i.e. the instruction
12724	* one, in order to avoid event nesting.
12725	*/
12726	PVM pVM = pVCpu->CTX_SUFF(pVM);
12727	if ( enmEvent1 != DBGFEVENT_END
12728	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12729	{
12730	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12731	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12732	if (rcStrict != VINF_SUCCESS)
12733	return rcStrict;
12734	}
12735	else if ( enmEvent2 != DBGFEVENT_END
12736	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12737	{
12738	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12739	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12740	if (rcStrict != VINF_SUCCESS)
12741	return rcStrict;
12742	}
12743
12744	return VINF_SUCCESS;
12745	}
12746
12747
12748	/**
12749	* Single-stepping VM-exit filtering.
12750	*
12751	* This is preprocessing the VM-exits and deciding whether we've gotten far
12752	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12753	* handling is performed.
12754	*
12755	* @returns Strict VBox status code (i.e. informational status codes too).
12756	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12757	* @param pVmxTransient The VMX-transient structure.
12758	* @param pDbgState The debug state.
12759	*/
12760	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12761	{
12762	/*
12763	* Expensive (saves context) generic dtrace VM-exit probe.
12764	*/
12765	uint32_t const uExitReason = pVmxTransient->uExitReason;
12766	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12767	{ /* more likely */ }
12768	else
12769	{
12770	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12771	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12772	AssertRC(rc);
12773	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12774	}
12775
12776	/*
12777	* Check for host NMI, just to get that out of the way.
12778	*/
12779	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12780	{ /* normally likely */ }
12781	else
12782	{
12783	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12784	AssertRCReturn(rc2, rc2);
12785	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12786	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12787	return hmR0VmxExitHostNmi(pVCpu);
12788	}
12789
12790	/*
12791	* Check for single stepping event if we're stepping.
12792	*/
12793	if (pVCpu->hm.s.fSingleInstruction)
12794	{
12795	switch (uExitReason)
12796	{
12797	case VMX_EXIT_MTF:
12798	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12799
12800	/* Various events: */
12801	case VMX_EXIT_XCPT_OR_NMI:
12802	case VMX_EXIT_EXT_INT:
12803	case VMX_EXIT_TRIPLE_FAULT:
12804	case VMX_EXIT_INT_WINDOW:
12805	case VMX_EXIT_NMI_WINDOW:
12806	case VMX_EXIT_TASK_SWITCH:
12807	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12808	case VMX_EXIT_APIC_ACCESS:
12809	case VMX_EXIT_EPT_VIOLATION:
12810	case VMX_EXIT_EPT_MISCONFIG:
12811	case VMX_EXIT_PREEMPT_TIMER:
12812
12813	/* Instruction specific VM-exits: */
12814	case VMX_EXIT_CPUID:
12815	case VMX_EXIT_GETSEC:
12816	case VMX_EXIT_HLT:
12817	case VMX_EXIT_INVD:
12818	case VMX_EXIT_INVLPG:
12819	case VMX_EXIT_RDPMC:
12820	case VMX_EXIT_RDTSC:
12821	case VMX_EXIT_RSM:
12822	case VMX_EXIT_VMCALL:
12823	case VMX_EXIT_VMCLEAR:
12824	case VMX_EXIT_VMLAUNCH:
12825	case VMX_EXIT_VMPTRLD:
12826	case VMX_EXIT_VMPTRST:
12827	case VMX_EXIT_VMREAD:
12828	case VMX_EXIT_VMRESUME:
12829	case VMX_EXIT_VMWRITE:
12830	case VMX_EXIT_VMXOFF:
12831	case VMX_EXIT_VMXON:
12832	case VMX_EXIT_MOV_CRX:
12833	case VMX_EXIT_MOV_DRX:
12834	case VMX_EXIT_IO_INSTR:
12835	case VMX_EXIT_RDMSR:
12836	case VMX_EXIT_WRMSR:
12837	case VMX_EXIT_MWAIT:
12838	case VMX_EXIT_MONITOR:
12839	case VMX_EXIT_PAUSE:
12840	case VMX_EXIT_GDTR_IDTR_ACCESS:
12841	case VMX_EXIT_LDTR_TR_ACCESS:
12842	case VMX_EXIT_INVEPT:
12843	case VMX_EXIT_RDTSCP:
12844	case VMX_EXIT_INVVPID:
12845	case VMX_EXIT_WBINVD:
12846	case VMX_EXIT_XSETBV:
12847	case VMX_EXIT_RDRAND:
12848	case VMX_EXIT_INVPCID:
12849	case VMX_EXIT_VMFUNC:
12850	case VMX_EXIT_RDSEED:
12851	case VMX_EXIT_XSAVES:
12852	case VMX_EXIT_XRSTORS:
12853	{
12854	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12855	AssertRCReturn(rc, rc);
12856	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12857	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12858	return VINF_EM_DBG_STEPPED;
12859	break;
12860	}
12861
12862	/* Errors and unexpected events: */
12863	case VMX_EXIT_INIT_SIGNAL:
12864	case VMX_EXIT_SIPI:
12865	case VMX_EXIT_IO_SMI:
12866	case VMX_EXIT_SMI:
12867	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12868	case VMX_EXIT_ERR_MSR_LOAD:
12869	case VMX_EXIT_ERR_MACHINE_CHECK:
12870	case VMX_EXIT_PML_FULL:
12871	case VMX_EXIT_VIRTUALIZED_EOI:
12872	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12873	break;
12874
12875	default:
12876	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12877	break;
12878	}
12879	}
12880
12881	/*
12882	* Check for debugger event breakpoints and dtrace probes.
12883	*/
12884	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12885	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12886	{
12887	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12888	if (rcStrict != VINF_SUCCESS)
12889	return rcStrict;
12890	}
12891
12892	/*
12893	* Normal processing.
12894	*/
12895	#ifdef HMVMX_USE_FUNCTION_TABLE
12896	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12897	#else
12898	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12899	#endif
12900	}
12901
12902
12903	/**
12904	* Single steps guest code using hardware-assisted VMX.
12905	*
12906	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12907	* but single-stepping through the hypervisor debugger.
12908	*
12909	* @returns Strict VBox status code (i.e. informational status codes too).
12910	* @param pVCpu The cross context virtual CPU structure.
12911	* @param pcLoops Pointer to the number of executed loops.
12912	*
12913	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12914	*/
12915	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12916	{
12917	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12918	Assert(pcLoops);
12919	Assert(*pcLoops <= cMaxResumeLoops);
12920
12921	VMXTRANSIENT VmxTransient;
12922	RT_ZERO(VmxTransient);
12923	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12924
12925	/* Set HMCPU indicators. */
12926	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12927	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12928	pVCpu->hm.s.fDebugWantRdTscExit = false;
12929	pVCpu->hm.s.fUsingDebugLoop = true;
12930
12931	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12932	VMXRUNDBGSTATE DbgState;
12933	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12934	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12935
12936	/*
12937	* The loop.
12938	*/
12939	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12940	for (;;)
12941	{
12942	Assert(!HMR0SuspendPending());
12943	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12944	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12945	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12946
12947	/* Set up VM-execution controls the next two can respond to. */
12948	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12949
12950	/*
12951	* Preparatory work for running guest code, this may force us to
12952	* return to ring-3.
12953	*
12954	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12955	*/
12956	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12957	if (rcStrict != VINF_SUCCESS)
12958	break;
12959
12960	/* Interrupts are disabled at this point! */
12961	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12962
12963	/* Override any obnoxious code in the above two calls. */
12964	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12965
12966	/*
12967	* Finally execute the guest.
12968	*/
12969	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12970
12971	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12972	/* Interrupts are re-enabled at this point! */
12973
12974	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12975	if (RT_SUCCESS(rcRun))
12976	{ /* very likely */ }
12977	else
12978	{
12979	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12980	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12981	return rcRun;
12982	}
12983
12984	/* Profile the VM-exit. */
12985	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12986	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12987	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12988	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12989	HMVMX_START_EXIT_DISPATCH_PROF();
12990
12991	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12992
12993	/*
12994	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12995	*/
12996	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12997	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12998	if (rcStrict != VINF_SUCCESS)
12999	break;
13000	if (++(*pcLoops) > cMaxResumeLoops)
13001	{
13002	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
13003	rcStrict = VINF_EM_RAW_INTERRUPT;
13004	break;
13005	}
13006
13007	/*
13008	* Stepping: Did the RIP change, if so, consider it a single step.
13009	* Otherwise, make sure one of the TFs gets set.
13010	*/
13011	if (fStepping)
13012	{
13013	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
13014	AssertRC(rc);
13015	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
13016	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
13017	{
13018	rcStrict = VINF_EM_DBG_STEPPED;
13019	break;
13020	}
13021	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
13022	}
13023
13024	/*
13025	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
13026	*/
13027	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
13028	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
13029	}
13030
13031	/*
13032	* Clear the X86_EFL_TF if necessary.
13033	*/
13034	if (pVCpu->hm.s.fClearTrapFlag)
13035	{
13036	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
13037	AssertRC(rc);
13038	pVCpu->hm.s.fClearTrapFlag = false;
13039	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
13040	}
13041	/** @todo there seems to be issues with the resume flag when the monitor trap
13042	* flag is pending without being used. Seen early in bios init when
13043	* accessing APIC page in protected mode. */
13044
13045	/*
13046	* Restore VM-exit control settings as we may not re-enter this function the
13047	* next time around.
13048	*/
13049	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
13050
13051	/* Restore HMCPU indicators. */
13052	pVCpu->hm.s.fUsingDebugLoop = false;
13053	pVCpu->hm.s.fDebugWantRdTscExit = false;
13054	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
13055
13056	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
13057	return rcStrict;
13058	}
13059
13060
13061	/** @} */
13062
13063
13064	/**
13065	* Checks if any expensive dtrace probes are enabled and we should go to the
13066	* debug loop.
13067	*
13068	* @returns true if we should use debug loop, false if not.
13069	*/
13070	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
13071	{
13072	/* It's probably faster to OR the raw 32-bit counter variables together.
13073	Since the variables are in an array and the probes are next to one
13074	another (more or less), we have good locality. So, better read
13075	eight-nine cache lines ever time and only have one conditional, than
13076	128+ conditionals, right? */
13077	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
13078	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
13079	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
13080	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
13081	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
13082	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
13083	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
13084	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
13085	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
13086	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
13087	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
13088	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
13089	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
13090	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
13091	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
13092	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
13093	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
13094	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
13095	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
13096	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
13097	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
13098	) != 0
13099	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
13100	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
13101	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
13102	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
13103	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
13104	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
13105	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
13106	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
13107	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
13108	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
13109	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
13110	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
13111	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
13112	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
13113	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
13114	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
13115	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
13116	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
13117	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
13118	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
13119	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
13120	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
13121	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
13122	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
13123	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
13124	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
13125	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
13126	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
13127	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
13128	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13129	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13130	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13131	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13132	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13133	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13134	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13135	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13136	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13137	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13138	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13139	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13140	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13141	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13142	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13143	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13144	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13145	) != 0
13146	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13147	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
13148	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13149	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13150	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13151	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
13152	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13153	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13154	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13155	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13156	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13157	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13158	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13159	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13160	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13161	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13162	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13163	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13164	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13165	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13166	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13167	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13168	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13169	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13170	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13171	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
13172	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
13173	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13174	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
13175	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13176	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13177	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13178	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13179	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13180	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13181	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13182	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13183	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13184	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13185	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13186	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13187	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13188	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13189	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13190	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13191	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13192	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13193	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13194	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13195	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13196	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13197	) != 0;
13198	}
13199
13200
13201	/**
13202	* Runs the guest using hardware-assisted VMX.
13203	*
13204	* @returns Strict VBox status code (i.e. informational status codes too).
13205	* @param pVCpu The cross context virtual CPU structure.
13206	*/
13207	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13208	{
13209	AssertPtr(pVCpu);
13210	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13211	Assert(VMMRZCallRing3IsEnabled(pVCpu));
13212	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13213	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13214
13215	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13216
13217	VBOXSTRICTRC rcStrict;
13218	uint32_t cLoops = 0;
13219	for (;;)
13220	{
13221	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13222	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13223	#else
13224	bool const fInNestedGuestMode = false;
13225	#endif
13226	if (!fInNestedGuestMode)
13227	{
13228	if ( !pVCpu->hm.s.fUseDebugLoop
13229	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
13230	&& !DBGFIsStepping(pVCpu)
13231	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13232	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13233	else
13234	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13235	}
13236	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13237	else
13238	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13239
13240	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13241	{
13242	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
13243	continue;
13244	}
13245	if (rcStrict == VINF_VMX_VMEXIT)
13246	{
13247	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
13248	continue;
13249	}
13250	#endif
13251	break;
13252	}
13253
13254	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13255	switch (rcLoop)
13256	{
13257	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13258	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13259	}
13260
13261	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13262	if (RT_FAILURE(rc2))
13263	{
13264	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13265	rcStrict = rc2;
13266	}
13267	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13268	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13269	return rcStrict;
13270	}
13271
13272
13273	#ifndef HMVMX_USE_FUNCTION_TABLE
13274	/**
13275	* Handles a guest VM-exit from hardware-assisted VMX execution.
13276	*
13277	* @returns Strict VBox status code (i.e. informational status codes too).
13278	* @param pVCpu The cross context virtual CPU structure.
13279	* @param pVmxTransient The VMX-transient structure.
13280	*/
13281	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13282	{
13283	#ifdef DEBUG_ramshankar
13284	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13285	do { \
13286	if (a_fSave != 0) \
13287	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
13288	VBOXSTRICTRC rcStrict = a_CallExpr; \
13289	if (a_fSave != 0) \
13290	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13291	return rcStrict; \
13292	} while (0)
13293	#else
13294	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13295	#endif
13296	uint32_t const uExitReason = pVmxTransient->uExitReason;
13297	switch (uExitReason)
13298	{
13299	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13300	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13301	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13302	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13303	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13304	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13305	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13306	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13307	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13308	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13309	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13310	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13311	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13312	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13313	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13314	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13315	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13316	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13317	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13318	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13319	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13320	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13321	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13322	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13323	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13324	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13325	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13326	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13327	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13328	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13329	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13330	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13331	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13332	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13333	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13334	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13335	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13336	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13337	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13338	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13339	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13340	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13341	#else
13342	case VMX_EXIT_VMCLEAR:
13343	case VMX_EXIT_VMLAUNCH:
13344	case VMX_EXIT_VMPTRLD:
13345	case VMX_EXIT_VMPTRST:
13346	case VMX_EXIT_VMREAD:
13347	case VMX_EXIT_VMRESUME:
13348	case VMX_EXIT_VMWRITE:
13349	case VMX_EXIT_VMXOFF:
13350	case VMX_EXIT_VMXON:
13351	case VMX_EXIT_INVVPID:
13352	case VMX_EXIT_INVEPT:
13353	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13354	#endif
13355
13356	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13357	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13358	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13359
13360	case VMX_EXIT_INIT_SIGNAL:
13361	case VMX_EXIT_SIPI:
13362	case VMX_EXIT_IO_SMI:
13363	case VMX_EXIT_SMI:
13364	case VMX_EXIT_ERR_MSR_LOAD:
13365	case VMX_EXIT_ERR_MACHINE_CHECK:
13366	case VMX_EXIT_PML_FULL:
13367	case VMX_EXIT_VIRTUALIZED_EOI:
13368	case VMX_EXIT_GDTR_IDTR_ACCESS:
13369	case VMX_EXIT_LDTR_TR_ACCESS:
13370	case VMX_EXIT_APIC_WRITE:
13371	case VMX_EXIT_RDRAND:
13372	case VMX_EXIT_RSM:
13373	case VMX_EXIT_VMFUNC:
13374	case VMX_EXIT_ENCLS:
13375	case VMX_EXIT_RDSEED:
13376	case VMX_EXIT_XSAVES:
13377	case VMX_EXIT_XRSTORS:
13378	case VMX_EXIT_UMWAIT:
13379	case VMX_EXIT_TPAUSE:
13380	default:
13381	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13382	}
13383	#undef VMEXIT_CALL_RET
13384	}
13385	#endif /* !HMVMX_USE_FUNCTION_TABLE */
13386
13387
13388	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13389	/**
13390	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13391	*
13392	* @returns Strict VBox status code (i.e. informational status codes too).
13393	* @param pVCpu The cross context virtual CPU structure.
13394	* @param pVmxTransient The VMX-transient structure.
13395	*/
13396	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13397	{
13398	uint32_t const uExitReason = pVmxTransient->uExitReason;
13399	switch (uExitReason)
13400	{
13401	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13402	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13403	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13404	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13405	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13406
13407	/*
13408	* We shouldn't direct host physical interrupts to the nested-guest.
13409	*/
13410	case VMX_EXIT_EXT_INT:
13411	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13412
13413	/*
13414	* Instructions that cause VM-exits unconditionally or the condition is
13415	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13416	* happens, it's guaranteed to be a nested-guest VM-exit).
13417	*
13418	* - Provides VM-exit instruction length ONLY.
13419	*/
13420	case VMX_EXIT_CPUID: /* Unconditional. */
13421	case VMX_EXIT_VMCALL:
13422	case VMX_EXIT_GETSEC:
13423	case VMX_EXIT_INVD:
13424	case VMX_EXIT_XSETBV:
13425	case VMX_EXIT_VMLAUNCH:
13426	case VMX_EXIT_VMRESUME:
13427	case VMX_EXIT_VMXOFF:
13428	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13429	case VMX_EXIT_VMFUNC:
13430	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13431
13432	/*
13433	* Instructions that cause VM-exits unconditionally or the condition is
13434	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13435	* happens, it's guaranteed to be a nested-guest VM-exit).
13436	*
13437	* - Provides VM-exit instruction length.
13438	* - Provides VM-exit information.
13439	* - Optionally provides Exit qualification.
13440	*
13441	* Since Exit qualification is 0 for all VM-exits where it is not
13442	* applicable, reading and passing it to the guest should produce
13443	* defined behavior.
13444	*
13445	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
13446	*/
13447	case VMX_EXIT_INVEPT: /* Unconditional. */
13448	case VMX_EXIT_INVVPID:
13449	case VMX_EXIT_VMCLEAR:
13450	case VMX_EXIT_VMPTRLD:
13451	case VMX_EXIT_VMPTRST:
13452	case VMX_EXIT_VMXON:
13453	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13454	case VMX_EXIT_LDTR_TR_ACCESS:
13455	case VMX_EXIT_RDRAND:
13456	case VMX_EXIT_RDSEED:
13457	case VMX_EXIT_XSAVES:
13458	case VMX_EXIT_XRSTORS:
13459	case VMX_EXIT_UMWAIT:
13460	case VMX_EXIT_TPAUSE:
13461	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13462
13463	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13464	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13465	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13466	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13467	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13468	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13469	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13470	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13471	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13472	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13473	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13474	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13475	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13476	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13477	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13478	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13479	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13480	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13481	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13482
13483	case VMX_EXIT_PREEMPT_TIMER:
13484	{
13485	/** @todo NSTVMX: Preempt timer. */
13486	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
13487	}
13488
13489	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13490	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13491
13492	case VMX_EXIT_VMREAD:
13493	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13494
13495	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13496	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13497
13498	case VMX_EXIT_INIT_SIGNAL:
13499	case VMX_EXIT_SIPI:
13500	case VMX_EXIT_IO_SMI:
13501	case VMX_EXIT_SMI:
13502	case VMX_EXIT_ERR_MSR_LOAD:
13503	case VMX_EXIT_ERR_MACHINE_CHECK:
13504	case VMX_EXIT_PML_FULL:
13505	case VMX_EXIT_RSM:
13506	default:
13507	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13508	}
13509	}
13510	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13511
13512
13513	/** @name VM-exit helpers.
13514	* @{
13515	*/
13516	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13517	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13518	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13519
13520	/** Macro for VM-exits called unexpectedly. */
13521	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13522	do { \
13523	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13524	return VERR_VMX_UNEXPECTED_EXIT; \
13525	} while (0)
13526
13527	#ifdef VBOX_STRICT
13528	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13529	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13530	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13531
13532	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13533	do { \
13534	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13535	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13536	} while (0)
13537
13538	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13539	do { \
13540	AssertPtr((a_pVCpu)); \
13541	AssertPtr((a_pVmxTransient)); \
13542	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13543	Assert((a_pVmxTransient)->pVmcsInfo); \
13544	Assert(ASMIntAreEnabled()); \
13545	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13546	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13547	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13548	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13549	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13550	HMVMX_ASSERT_PREEMPT_CPUID(); \
13551	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13552	} while (0)
13553
13554	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13555	do { \
13556	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13557	Assert((a_pVmxTransient)->fIsNestedGuest); \
13558	} while (0)
13559
13560	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13561	do { \
13562	Log4Func(("\n")); \
13563	} while (0)
13564	#else
13565	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13566	do { \
13567	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13568	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13569	} while (0)
13570
13571	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13572	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13573
13574	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13575	#endif
13576
13577	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13578	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13579	* guests that attempted to execute a VMX instruction. */
13580	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13581	do \
13582	{ \
13583	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13584	if (rcStrictTmp == VINF_SUCCESS) \
13585	{ /* likely */ } \
13586	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13587	{ \
13588	Assert((a_pVCpu)->hm.s.Event.fPending); \
13589	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13590	return VINF_SUCCESS; \
13591	} \
13592	else \
13593	{ \
13594	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13595	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13596	} \
13597	} while (0)
13598
13599	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13600	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13601	do \
13602	{ \
13603	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13604	(a_pGCPtrEffAddr)); \
13605	if (rcStrictTmp == VINF_SUCCESS) \
13606	{ /* likely */ } \
13607	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13608	{ \
13609	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13610	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13611	NOREF(uXcptTmp); \
13612	return VINF_SUCCESS; \
13613	} \
13614	else \
13615	{ \
13616	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13617	return rcStrictTmp; \
13618	} \
13619	} while (0)
13620	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13621
13622
13623	/**
13624	* Advances the guest RIP by the specified number of bytes.
13625	*
13626	* @param pVCpu The cross context virtual CPU structure.
13627	* @param cbInstr Number of bytes to advance the RIP by.
13628	*
13629	* @remarks No-long-jump zone!!!
13630	*/
13631	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13632	{
13633	/* Advance the RIP. */
13634	pVCpu->cpum.GstCtx.rip += cbInstr;
13635	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13636
13637	/* Update interrupt inhibition. */
13638	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13639	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13640	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13641	}
13642
13643
13644	/**
13645	* Advances the guest RIP after reading it from the VMCS.
13646	*
13647	* @returns VBox status code, no informational status codes.
13648	* @param pVCpu The cross context virtual CPU structure.
13649	* @param pVmxTransient The VMX-transient structure.
13650	*
13651	* @remarks No-long-jump zone!!!
13652	*/
13653	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13654	{
13655	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13656	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13657	AssertRCReturn(rc, rc);
13658
13659	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13660	return VINF_SUCCESS;
13661	}
13662
13663
13664	/**
13665	* Handle a condition that occurred while delivering an event through the guest
13666	* IDT.
13667	*
13668	* @returns Strict VBox status code (i.e. informational status codes too).
13669	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13670	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13671	* to continue execution of the guest which will delivery the \#DF.
13672	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13673	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13674	*
13675	* @param pVCpu The cross context virtual CPU structure.
13676	* @param pVmxTransient The VMX-transient structure.
13677	*
13678	* @remarks No-long-jump zone!!!
13679	*/
13680	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13681	{
13682	/* Read the IDT vectoring info. and VM-exit interruption info. */
13683	{
13684	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13685	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13686	AssertRCReturn(rc, rc);
13687	}
13688
13689	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13690	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13691	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13692	{
13693	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13694	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13695
13696	/*
13697	* If the event was a software interrupt (generated with INT n) or a software exception
13698	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13699	* can handle the VM-exit and continue guest execution which will re-execute the
13700	* instruction rather than re-injecting the exception, as that can cause premature
13701	* trips to ring-3 before injection and involve TRPM which currently has no way of
13702	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13703	* the problem).
13704	*/
13705	IEMXCPTRAISE enmRaise;
13706	IEMXCPTRAISEINFO fRaiseInfo;
13707	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13708	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13709	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13710	{
13711	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13712	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13713	}
13714	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13715	{
13716	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13717	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13718	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13719
13720	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13721	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13722	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13723
13724	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13725
13726	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13727	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13728	{
13729	pVmxTransient->fVectoringPF = true;
13730	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13731	}
13732	}
13733	else
13734	{
13735	/*
13736	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13737	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13738	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13739	*/
13740	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13741	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13742	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13743	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13744	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13745	}
13746
13747	/*
13748	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13749	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13750	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13751	* subsequent VM-entry would fail, see @bugref{7445}.
13752	*
13753	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13754	*/
13755	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13756	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13757	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13758	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13759	&& CPUMIsGuestNmiBlocking(pVCpu))
13760	{
13761	CPUMSetGuestNmiBlocking(pVCpu, false);
13762	}
13763
13764	switch (enmRaise)
13765	{
13766	case IEMXCPTRAISE_CURRENT_XCPT:
13767	{
13768	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13769	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13770	Assert(rcStrict == VINF_SUCCESS);
13771	break;
13772	}
13773
13774	case IEMXCPTRAISE_PREV_EVENT:
13775	{
13776	uint32_t u32ErrCode;
13777	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13778	{
13779	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13780	AssertRCReturn(rc, rc);
13781	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13782	}
13783	else
13784	u32ErrCode = 0;
13785
13786	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13787	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13788	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13789	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13790
13791	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13792	pVCpu->hm.s.Event.u32ErrCode));
13793	Assert(rcStrict == VINF_SUCCESS);
13794	break;
13795	}
13796
13797	case IEMXCPTRAISE_REEXEC_INSTR:
13798	Assert(rcStrict == VINF_SUCCESS);
13799	break;
13800
13801	case IEMXCPTRAISE_DOUBLE_FAULT:
13802	{
13803	/*
13804	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13805	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13806	*/
13807	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13808	{
13809	pVmxTransient->fVectoringDoublePF = true;
13810	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13811	pVCpu->cpum.GstCtx.cr2));
13812	rcStrict = VINF_SUCCESS;
13813	}
13814	else
13815	{
13816	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
13817	hmR0VmxSetPendingXcptDF(pVCpu);
13818	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13819	uIdtVector, uExitVector));
13820	rcStrict = VINF_HM_DOUBLE_FAULT;
13821	}
13822	break;
13823	}
13824
13825	case IEMXCPTRAISE_TRIPLE_FAULT:
13826	{
13827	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13828	rcStrict = VINF_EM_RESET;
13829	break;
13830	}
13831
13832	case IEMXCPTRAISE_CPU_HANG:
13833	{
13834	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13835	rcStrict = VERR_EM_GUEST_CPU_HANG;
13836	break;
13837	}
13838
13839	default:
13840	{
13841	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13842	rcStrict = VERR_VMX_IPE_2;
13843	break;
13844	}
13845	}
13846	}
13847	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13848	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13849	&& uExitVector != X86_XCPT_DF
13850	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13851	{
13852	/*
13853	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13854	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13855	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13856	*/
13857	CPUMSetGuestNmiBlocking(pVCpu, true);
13858	Log4Func(("Set NMI blocking. fValid=%RTbool uExitReason=%u\n", VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo),
13859	pVmxTransient->uExitReason));
13860	}
13861
13862	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13863	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13864	return rcStrict;
13865	}
13866
13867
13868	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13869	/**
13870	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13871	* guest attempting to execute a VMX instruction.
13872	*
13873	* @returns Strict VBox status code (i.e. informational status codes too).
13874	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13875	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13876	*
13877	* @param pVCpu The cross context virtual CPU structure.
13878	* @param uExitReason The VM-exit reason.
13879	*
13880	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13881	* @remarks No-long-jump zone!!!
13882	*/
13883	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13884	{
13885	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13886	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13887
13888	/*
13889	* The physical CPU would have already checked the CPU mode/code segment.
13890	* We shall just assert here for paranoia.
13891	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13892	*/
13893	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13894	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13895	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13896
13897	if (uExitReason == VMX_EXIT_VMXON)
13898	{
13899	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13900
13901	/*
13902	* We check CR4.VMXE because it is required to be always set while in VMX operation
13903	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13904	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13905	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13906	*/
13907	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13908	{
13909	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13910	hmR0VmxSetPendingXcptUD(pVCpu);
13911	return VINF_HM_PENDING_XCPT;
13912	}
13913	}
13914	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13915	{
13916	/*
13917	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13918	* (other than VMXON), we need to raise a #UD.
13919	*/
13920	Log4Func(("Not in VMX root mode -> #UD\n"));
13921	hmR0VmxSetPendingXcptUD(pVCpu);
13922	return VINF_HM_PENDING_XCPT;
13923	}
13924
13925	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13926	return VINF_SUCCESS;
13927	}
13928
13929	/**
13930	* Decodes the memory operand of an instruction that caused a VM-exit.
13931	*
13932	* The Exit qualification field provides the displacement field for memory
13933	* operand instructions, if any.
13934	*
13935	* @returns Strict VBox status code (i.e. informational status codes too).
13936	* @retval VINF_SUCCESS if the operand was successfully decoded.
13937	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13938	* operand.
13939	* @param pVCpu The cross context virtual CPU structure.
13940	* @param uExitInstrInfo The VM-exit instruction information field.
13941	* @param enmMemAccess The memory operand's access type (read or write).
13942	* @param GCPtrDisp The instruction displacement field, if any. For
13943	* RIP-relative addressing pass RIP + displacement here.
13944	* @param pGCPtrMem Where to store the effective destination memory address.
13945	*
13946	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13947	* virtual-8086 mode hence skips those checks while verifying if the
13948	* segment is valid.
13949	*/
13950	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13951	PRTGCPTR pGCPtrMem)
13952	{
13953	Assert(pGCPtrMem);
13954	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13955	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13956	\| CPUMCTX_EXTRN_CR0);
13957
13958	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13959	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13960	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13961
13962	VMXEXITINSTRINFO ExitInstrInfo;
13963	ExitInstrInfo.u = uExitInstrInfo;
13964	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13965	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13966	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13967	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13968	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13969	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13970	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13971	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13972	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13973
13974	/*
13975	* Validate instruction information.
13976	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13977	*/
13978	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13979	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13980	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13981	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13982	AssertLogRelMsgReturn(fIsMemOperand,
13983	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13984
13985	/*
13986	* Compute the complete effective address.
13987	*
13988	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13989	* See AMD spec. 4.5.2 "Segment Registers".
13990	*/
13991	RTGCPTR GCPtrMem = GCPtrDisp;
13992	if (fBaseRegValid)
13993	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13994	if (fIdxRegValid)
13995	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13996
13997	RTGCPTR const GCPtrOff = GCPtrMem;
13998	if ( !fIsLongMode
13999	\|\| iSegReg >= X86_SREG_FS)
14000	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
14001	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
14002
14003	/*
14004	* Validate effective address.
14005	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
14006	*/
14007	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
14008	Assert(cbAccess > 0);
14009	if (fIsLongMode)
14010	{
14011	if (X86_IS_CANONICAL(GCPtrMem))
14012	{
14013	*pGCPtrMem = GCPtrMem;
14014	return VINF_SUCCESS;
14015	}
14016
14017	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
14018	* "Data Limit Checks in 64-bit Mode". */
14019	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
14020	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14021	return VINF_HM_PENDING_XCPT;
14022	}
14023
14024	/*
14025	* This is a watered down version of iemMemApplySegment().
14026	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
14027	* and segment CPL/DPL checks are skipped.
14028	*/
14029	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
14030	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
14031	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
14032
14033	/* Check if the segment is present and usable. */
14034	if ( pSel->Attr.n.u1Present
14035	&& !pSel->Attr.n.u1Unusable)
14036	{
14037	Assert(pSel->Attr.n.u1DescType);
14038	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
14039	{
14040	/* Check permissions for the data segment. */
14041	if ( enmMemAccess == VMXMEMACCESS_WRITE
14042	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
14043	{
14044	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14045	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
14046	return VINF_HM_PENDING_XCPT;
14047	}
14048
14049	/* Check limits if it's a normal data segment. */
14050	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
14051	{
14052	if ( GCPtrFirst32 > pSel->u32Limit
14053	\|\| GCPtrLast32 > pSel->u32Limit)
14054	{
14055	Log4Func(("Data segment limit exceeded. "
14056	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14057	GCPtrLast32, pSel->u32Limit));
14058	if (iSegReg == X86_SREG_SS)
14059	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14060	else
14061	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14062	return VINF_HM_PENDING_XCPT;
14063	}
14064	}
14065	else
14066	{
14067	/* Check limits if it's an expand-down data segment.
14068	Note! The upper boundary is defined by the B bit, not the G bit! */
14069	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
14070	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
14071	{
14072	Log4Func(("Expand-down data segment limit exceeded. "
14073	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
14074	GCPtrLast32, pSel->u32Limit));
14075	if (iSegReg == X86_SREG_SS)
14076	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14077	else
14078	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14079	return VINF_HM_PENDING_XCPT;
14080	}
14081	}
14082	}
14083	else
14084	{
14085	/* Check permissions for the code segment. */
14086	if ( enmMemAccess == VMXMEMACCESS_WRITE
14087	\|\| ( enmMemAccess == VMXMEMACCESS_READ
14088	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
14089	{
14090	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
14091	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
14092	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14093	return VINF_HM_PENDING_XCPT;
14094	}
14095
14096	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
14097	if ( GCPtrFirst32 > pSel->u32Limit
14098	\|\| GCPtrLast32 > pSel->u32Limit)
14099	{
14100	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
14101	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
14102	if (iSegReg == X86_SREG_SS)
14103	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14104	else
14105	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14106	return VINF_HM_PENDING_XCPT;
14107	}
14108	}
14109	}
14110	else
14111	{
14112	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14113	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14114	return VINF_HM_PENDING_XCPT;
14115	}
14116
14117	*pGCPtrMem = GCPtrMem;
14118	return VINF_SUCCESS;
14119	}
14120	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14121
14122
14123	/**
14124	* VM-exit helper for LMSW.
14125	*/
14126	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
14127	{
14128	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14129	AssertRCReturn(rc, rc);
14130
14131	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14132	AssertMsg( rcStrict == VINF_SUCCESS
14133	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14134
14135	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14136	if (rcStrict == VINF_IEM_RAISED_XCPT)
14137	{
14138	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14139	rcStrict = VINF_SUCCESS;
14140	}
14141
14142	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14143	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14144	return rcStrict;
14145	}
14146
14147
14148	/**
14149	* VM-exit helper for CLTS.
14150	*/
14151	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
14152	{
14153	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14154	AssertRCReturn(rc, rc);
14155
14156	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
14157	AssertMsg( rcStrict == VINF_SUCCESS
14158	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14159
14160	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14161	if (rcStrict == VINF_IEM_RAISED_XCPT)
14162	{
14163	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14164	rcStrict = VINF_SUCCESS;
14165	}
14166
14167	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14168	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14169	return rcStrict;
14170	}
14171
14172
14173	/**
14174	* VM-exit helper for MOV from CRx (CRx read).
14175	*/
14176	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14177	{
14178	Assert(iCrReg < 16);
14179	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14180
14181	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14182	AssertRCReturn(rc, rc);
14183
14184	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
14185	AssertMsg( rcStrict == VINF_SUCCESS
14186	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14187
14188	if (iGReg == X86_GREG_xSP)
14189	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14190	else
14191	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14192	#ifdef VBOX_WITH_STATISTICS
14193	switch (iCrReg)
14194	{
14195	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14196	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14197	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14198	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14199	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14200	}
14201	#endif
14202	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
14203	return rcStrict;
14204	}
14205
14206
14207	/**
14208	* VM-exit helper for MOV to CRx (CRx write).
14209	*/
14210	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14211	{
14212	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14213	AssertRCReturn(rc, rc);
14214
14215	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
14216	AssertMsg( rcStrict == VINF_SUCCESS
14217	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14218	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14219
14220	switch (iCrReg)
14221	{
14222	case 0:
14223	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14225	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14226	break;
14227
14228	case 2:
14229	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14230	/* Nothing to do here, CR2 it's not part of the VMCS. */
14231	break;
14232
14233	case 3:
14234	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14235	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14236	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14237	break;
14238
14239	case 4:
14240	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14241	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14242	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14243	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14244	break;
14245
14246	case 8:
14247	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14248	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14249	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14250	break;
14251
14252	default:
14253	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
14254	break;
14255	}
14256
14257	if (rcStrict == VINF_IEM_RAISED_XCPT)
14258	{
14259	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14260	rcStrict = VINF_SUCCESS;
14261	}
14262	return rcStrict;
14263	}
14264
14265
14266	/**
14267	* VM-exit exception handler for \#PF (Page-fault exception).
14268	*/
14269	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14270	{
14271	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14272	PVM pVM = pVCpu->CTX_SUFF(pVM);
14273	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14274	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14275	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14276	AssertRCReturn(rc, rc);
14277
14278	if (!pVM->hm.s.fNestedPaging)
14279	{ /* likely */ }
14280	else
14281	{
14282	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14283	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
14284	#endif
14285	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14286	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14287	{
14288	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14289	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
14290	}
14291	else
14292	{
14293	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14294	hmR0VmxSetPendingXcptDF(pVCpu);
14295	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
14296	}
14297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14298	return rc;
14299	}
14300
14301	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14302	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14303	if (pVmxTransient->fVectoringPF)
14304	{
14305	Assert(pVCpu->hm.s.Event.fPending);
14306	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14307	}
14308
14309	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14310	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14311	AssertRCReturn(rc, rc);
14312
14313	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
14314	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
14315
14316	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14317	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
14318
14319	Log4Func(("#PF: rc=%Rrc\n", rc));
14320	if (rc == VINF_SUCCESS)
14321	{
14322	/*
14323	* This is typically a shadow page table sync or a MMIO instruction. But we may have
14324	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14325	*/
14326	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14327	TRPMResetTrap(pVCpu);
14328	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14329	return rc;
14330	}
14331
14332	if (rc == VINF_EM_RAW_GUEST_TRAP)
14333	{
14334	if (!pVmxTransient->fVectoringDoublePF)
14335	{
14336	/* It's a guest page fault and needs to be reflected to the guest. */
14337	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14338	TRPMResetTrap(pVCpu);
14339	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14340	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14341	uGstErrorCode, pVmxTransient->uExitQual);
14342	}
14343	else
14344	{
14345	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14346	TRPMResetTrap(pVCpu);
14347	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14348	hmR0VmxSetPendingXcptDF(pVCpu);
14349	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
14350	}
14351
14352	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14353	return VINF_SUCCESS;
14354	}
14355
14356	TRPMResetTrap(pVCpu);
14357	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14358	return rc;
14359	}
14360
14361
14362	/**
14363	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14364	*/
14365	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14366	{
14367	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14369
14370	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14371	AssertRCReturn(rc, rc);
14372
14373	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14374	{
14375	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14376	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14377
14378	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14379	* provides VM-exit instruction length. If this causes problem later,
14380	* disassemble the instruction like it's done on AMD-V. */
14381	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14382	AssertRCReturn(rc2, rc2);
14383	return rc;
14384	}
14385
14386	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14387	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14388	return rc;
14389	}
14390
14391
14392	/**
14393	* VM-exit exception handler for \#BP (Breakpoint exception).
14394	*/
14395	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14396	{
14397	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14398	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14399
14400	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14401	AssertRCReturn(rc, rc);
14402
14403	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14404	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14405	if (rc == VINF_EM_RAW_GUEST_TRAP)
14406	{
14407	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14408	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14409	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14410	AssertRCReturn(rc, rc);
14411
14412	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14413	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14414	}
14415
14416	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14417	return rc;
14418	}
14419
14420
14421	/**
14422	* VM-exit exception handler for \#AC (alignment check exception).
14423	*/
14424	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14425	{
14426	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14427	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
14428
14429	/*
14430	* Re-inject it. We'll detect any nesting before getting here.
14431	*/
14432	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14433	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14434	AssertRCReturn(rc, rc);
14435	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14436
14437	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14438	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14439	return VINF_SUCCESS;
14440	}
14441
14442
14443	/**
14444	* VM-exit exception handler for \#DB (Debug exception).
14445	*/
14446	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14447	{
14448	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14450
14451	/*
14452	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
14453	*/
14454	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14455
14456	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14457	uint64_t const uDR6 = X86_DR6_INIT_VAL
14458	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14459	\| X86_DR6_BD \| X86_DR6_BS));
14460
14461	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14462	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14463	Log6Func(("rc=%Rrc\n", rc));
14464	if (rc == VINF_EM_RAW_GUEST_TRAP)
14465	{
14466	/*
14467	* The exception was for the guest. Update DR6, DR7.GD and
14468	* IA32_DEBUGCTL.LBR before forwarding it.
14469	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14470	*/
14471	VMMRZCallRing3Disable(pVCpu);
14472	HM_DISABLE_PREEMPT(pVCpu);
14473
14474	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14475	pCtx->dr[6] \|= uDR6;
14476	if (CPUMIsGuestDebugStateActive(pVCpu))
14477	ASMSetDR6(pCtx->dr[6]);
14478
14479	HM_RESTORE_PREEMPT();
14480	VMMRZCallRing3Enable(pVCpu);
14481
14482	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14483	AssertRCReturn(rc, rc);
14484
14485	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14486	pCtx->dr[7] &= ~X86_DR7_GD;
14487
14488	/* Paranoia. */
14489	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14490	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14491
14492	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14493	AssertRCReturn(rc, rc);
14494
14495	/*
14496	* Raise #DB in the guest.
14497	*
14498	* It is important to reflect exactly what the VM-exit gave us (preserving the
14499	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14500	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14501	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14502	*
14503	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14504	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14505	*/
14506	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14507	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14508	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14509	AssertRCReturn(rc, rc);
14510	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14511	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14512	return VINF_SUCCESS;
14513	}
14514
14515	/*
14516	* Not a guest trap, must be a hypervisor related debug event then.
14517	* Update DR6 in case someone is interested in it.
14518	*/
14519	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14520	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14521	CPUMSetHyperDR6(pVCpu, uDR6);
14522
14523	return rc;
14524	}
14525
14526
14527	/**
14528	* Hacks its way around the lovely mesa driver's backdoor accesses.
14529	*
14530	* @sa hmR0SvmHandleMesaDrvGp.
14531	*/
14532	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14533	{
14534	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14535	RT_NOREF(pCtx);
14536
14537	/* For now we'll just skip the instruction. */
14538	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14539	}
14540
14541
14542	/**
14543	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14544	* backdoor logging w/o checking what it is running inside.
14545	*
14546	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14547	* backdoor port and magic numbers loaded in registers.
14548	*
14549	* @returns true if it is, false if it isn't.
14550	* @sa hmR0SvmIsMesaDrvGp.
14551	*/
14552	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14553	{
14554	/* 0xed: IN eAX,dx */
14555	uint8_t abInstr[1];
14556	if (pVmxTransient->cbInstr != sizeof(abInstr))
14557	return false;
14558
14559	/* Check that it is #GP(0). */
14560	if (pVmxTransient->uExitIntErrorCode != 0)
14561	return false;
14562
14563	/* Check magic and port. */
14564	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14565	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14566	if (pCtx->rax != UINT32_C(0x564d5868))
14567	return false;
14568	if (pCtx->dx != UINT32_C(0x5658))
14569	return false;
14570
14571	/* Flat ring-3 CS. */
14572	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14573	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14574	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14575	if (pCtx->cs.Attr.n.u2Dpl != 3)
14576	return false;
14577	if (pCtx->cs.u64Base != 0)
14578	return false;
14579
14580	/* Check opcode. */
14581	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14582	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14583	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14584	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14585	if (RT_FAILURE(rc))
14586	return false;
14587	if (abInstr[0] != 0xed)
14588	return false;
14589
14590	return true;
14591	}
14592
14593	/**
14594	* VM-exit exception handler for \#GP (General-protection exception).
14595	*
14596	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14597	*/
14598	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14599	{
14600	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14602
14603	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14604	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14605	if (pVmcsInfo->RealMode.fRealOnV86Active)
14606	{ /* likely */ }
14607	else
14608	{
14609	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14610	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14611	#endif
14612	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14613	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14614	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14615	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14616	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14617	AssertRCReturn(rc, rc);
14618	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14619	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14620
14621	if ( pVmxTransient->fIsNestedGuest
14622	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14623	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14624	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14625	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14626	else
14627	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14628	return rc;
14629	}
14630
14631	Assert(CPUMIsGuestInRealModeEx(pCtx));
14632	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14633	Assert(!pVmxTransient->fIsNestedGuest);
14634
14635	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14636	AssertRCReturn(rc, rc);
14637
14638	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14639	if (rcStrict == VINF_SUCCESS)
14640	{
14641	if (!CPUMIsGuestInRealModeEx(pCtx))
14642	{
14643	/*
14644	* The guest is no longer in real-mode, check if we can continue executing the
14645	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14646	*/
14647	pVmcsInfo->RealMode.fRealOnV86Active = false;
14648	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14649	{
14650	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14651	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14652	}
14653	else
14654	{
14655	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14656	rcStrict = VINF_EM_RESCHEDULE;
14657	}
14658	}
14659	else
14660	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14661	}
14662	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14663	{
14664	rcStrict = VINF_SUCCESS;
14665	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14666	}
14667	return VBOXSTRICTRC_VAL(rcStrict);
14668	}
14669
14670
14671	/**
14672	* VM-exit exception handler wrapper for generic exceptions.
14673	*
14674	* This simply re-injects the exception back into the VM without any special
14675	* processing.
14676	*/
14677	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14678	{
14679	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14680
14681	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14682	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14683	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14684	("uVector=%#x u32XcptBitmap=%#X32\n",
14685	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14686	NOREF(pVmcsInfo);
14687	#endif
14688
14689	/*
14690	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14691	* would have been handled while checking exits due to event delivery.
14692	*/
14693	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14694	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14695	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14696	AssertRCReturn(rc, rc);
14697
14698	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14699
14700	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14701	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14702	AssertRCReturn(rc, rc);
14703	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14704	#endif
14705
14706	#ifdef VBOX_WITH_STATISTICS
14707	switch (uVector)
14708	{
14709	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14710	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14711	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14712	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14713	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14714	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14715	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14716	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14717	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14718	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14719	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14720	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14721	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14722	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14723	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14724	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14725	default:
14726	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14727	break;
14728	}
14729	#endif
14730
14731	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14732	Assert( uVector != X86_XCPT_PF
14733	\|\| VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo) != VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
14734	NOREF(uVector);
14735
14736	/* Re-inject the original exception into the guest. */
14737	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14738	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14739	return VINF_SUCCESS;
14740	}
14741	/** @} */
14742
14743
14744	/** @name VM-exit handlers.
14745	* @{
14746	*/
14747	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14748	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14749	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14750
14751	/**
14752	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14753	*/
14754	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14755	{
14756	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14758	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14759	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14760	return VINF_SUCCESS;
14761	return VINF_EM_RAW_INTERRUPT;
14762	}
14763
14764
14765	/**
14766	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14767	* VM-exit.
14768	*/
14769	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14770	{
14771	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14772	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14773
14774	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14775	AssertRCReturn(rc, rc);
14776
14777	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14778	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14779
14780	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14781	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14782	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14783	NOREF(pVmcsInfo);
14784
14785	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
14786	{
14787	/*
14788	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14789	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14790	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14791	*
14792	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14793	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
14794	*/
14795	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14796	return hmR0VmxExitHostNmi(pVCpu);
14797	}
14798
14799	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14800	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14801	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
14802	{ /* likely */ }
14803	else
14804	{
14805	if (rcStrict == VINF_HM_DOUBLE_FAULT)
14806	rcStrict = VINF_SUCCESS;
14807	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14808	return rcStrict;
14809	}
14810
14811	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
14812	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
14813	switch (uIntType)
14814	{
14815	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
14816	Assert(uVector == X86_XCPT_DB);
14817	RT_FALL_THRU();
14818	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
14819	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14820	RT_FALL_THRU();
14821	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14822	{
14823	/*
14824	* If there's any exception caused as a result of event injection, the resulting
14825	* secondary/final execption will be pending, we shall continue guest execution
14826	* after injecting the event. The page-fault case is complicated and we manually
14827	* handle any currently pending event in hmR0VmxExitXcptPF.
14828	*/
14829	if (!pVCpu->hm.s.Event.fPending)
14830	{ /* likely */ }
14831	else if (uVector != X86_XCPT_PF)
14832	{
14833	rcStrict = VINF_SUCCESS;
14834	break;
14835	}
14836
14837	switch (uVector)
14838	{
14839	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
14840	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
14841	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
14842	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
14843	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
14844	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
14845	default:
14846	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient);
14847	break;
14848	}
14849	break;
14850	}
14851
14852	default:
14853	{
14854	pVCpu->hm.s.u32HMError = uExitIntInfo;
14855	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14856	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
14857	break;
14858	}
14859	}
14860	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14861	return rcStrict;
14862	}
14863
14864
14865	/**
14866	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14867	*/
14868	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14869	{
14870	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14871
14872	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14873	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14874	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14875	AssertRCReturn(rc, rc);
14876
14877	/* Evaluate and deliver pending events and resume guest execution. */
14878	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14879	return VINF_SUCCESS;
14880	}
14881
14882
14883	/**
14884	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14885	*/
14886	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14887	{
14888	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14889
14890	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14891	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14892	{
14893	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14894	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14895	}
14896
14897	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14898
14899	/*
14900	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14901	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14902	*/
14903	uint32_t fIntrState;
14904	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14905	AssertRCReturn(rc, rc);
14906	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14907	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14908	{
14909	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14910	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14911
14912	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14913	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14914	AssertRCReturn(rc, rc);
14915	}
14916
14917	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14918	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14919	AssertRCReturn(rc, rc);
14920
14921	/* Evaluate and deliver pending events and resume guest execution. */
14922	return VINF_SUCCESS;
14923	}
14924
14925
14926	/**
14927	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14928	*/
14929	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14930	{
14931	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14932	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14933	}
14934
14935
14936	/**
14937	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14938	*/
14939	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14940	{
14941	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14942	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14943	}
14944
14945
14946	/**
14947	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14948	*/
14949	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14950	{
14951	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14952
14953	/*
14954	* Get the state we need and update the exit history entry.
14955	*/
14956	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14957	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14958	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14959	AssertRCReturn(rc, rc);
14960
14961	VBOXSTRICTRC rcStrict;
14962	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14963	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14964	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14965	if (!pExitRec)
14966	{
14967	/*
14968	* Regular CPUID instruction execution.
14969	*/
14970	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14971	if (rcStrict == VINF_SUCCESS)
14972	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14973	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14974	{
14975	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14976	rcStrict = VINF_SUCCESS;
14977	}
14978	}
14979	else
14980	{
14981	/*
14982	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14983	*/
14984	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14985	AssertRCReturn(rc2, rc2);
14986
14987	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14988	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14989
14990	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14991	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14992
14993	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14994	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14995	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14996	}
14997	return rcStrict;
14998	}
14999
15000
15001	/**
15002	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
15003	*/
15004	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15005	{
15006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15007
15008	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15009	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
15010	AssertRCReturn(rc, rc);
15011
15012	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
15013	return VINF_EM_RAW_EMULATE_INSTR;
15014
15015	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
15016	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15017	}
15018
15019
15020	/**
15021	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
15022	*/
15023	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15024	{
15025	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15026
15027	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15028	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15029	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15030	AssertRCReturn(rc, rc);
15031
15032	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
15033	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15034	{
15035	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15036	we must reset offsetting on VM-entry. See @bugref{6634}. */
15037	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15038	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15039	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15040	}
15041	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15042	{
15043	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15044	rcStrict = VINF_SUCCESS;
15045	}
15046	return rcStrict;
15047	}
15048
15049
15050	/**
15051	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
15052	*/
15053	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15054	{
15055	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15056
15057	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15058	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
15059	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15060	AssertRCReturn(rc, rc);
15061
15062	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
15063	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15064	{
15065	/* If we get a spurious VM-exit when TSC offsetting is enabled,
15066	we must reset offsetting on VM-reentry. See @bugref{6634}. */
15067	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
15068	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15069	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15070	}
15071	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15072	{
15073	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15074	rcStrict = VINF_SUCCESS;
15075	}
15076	return rcStrict;
15077	}
15078
15079
15080	/**
15081	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15082	*/
15083	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15084	{
15085	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15086
15087	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15088	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
15089	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
15090	AssertRCReturn(rc, rc);
15091
15092	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15093	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15094	if (RT_LIKELY(rc == VINF_SUCCESS))
15095	{
15096	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15097	Assert(pVmxTransient->cbInstr == 2);
15098	}
15099	else
15100	{
15101	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
15102	rc = VERR_EM_INTERPRETER;
15103	}
15104	return rc;
15105	}
15106
15107
15108	/**
15109	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
15110	*/
15111	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15112	{
15113	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15114
15115	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
15116	if (EMAreHypercallInstructionsEnabled(pVCpu))
15117	{
15118	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15119	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
15120	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
15121	AssertRCReturn(rc, rc);
15122
15123	/* Perform the hypercall. */
15124	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
15125	if (rcStrict == VINF_SUCCESS)
15126	{
15127	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15128	AssertRCReturn(rc, rc);
15129	}
15130	else
15131	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
15132	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
15133	\|\| RT_FAILURE(rcStrict));
15134
15135	/* If the hypercall changes anything other than guest's general-purpose registers,
15136	we would need to reload the guest changed bits here before VM-entry. */
15137	}
15138	else
15139	Log4Func(("Hypercalls not enabled\n"));
15140
15141	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
15142	if (RT_FAILURE(rcStrict))
15143	{
15144	hmR0VmxSetPendingXcptUD(pVCpu);
15145	rcStrict = VINF_SUCCESS;
15146	}
15147
15148	return rcStrict;
15149	}
15150
15151
15152	/**
15153	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15154	*/
15155	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15156	{
15157	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15158	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
15159
15160	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15161	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15162	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15163	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15164	AssertRCReturn(rc, rc);
15165
15166	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
15167
15168	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
15169	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15170	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15171	{
15172	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15173	rcStrict = VINF_SUCCESS;
15174	}
15175	else
15176	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
15177	VBOXSTRICTRC_VAL(rcStrict)));
15178	return rcStrict;
15179	}
15180
15181
15182	/**
15183	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
15184	*/
15185	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15186	{
15187	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15188
15189	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15190	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
15191	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15192	AssertRCReturn(rc, rc);
15193
15194	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
15195	if (rcStrict == VINF_SUCCESS)
15196	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15197	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15198	{
15199	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15200	rcStrict = VINF_SUCCESS;
15201	}
15202
15203	return rcStrict;
15204	}
15205
15206
15207	/**
15208	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
15209	*/
15210	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15211	{
15212	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15213
15214	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15215	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15216	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15217	AssertRCReturn(rc, rc);
15218
15219	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
15220	if (RT_SUCCESS(rcStrict))
15221	{
15222	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15223	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
15224	rcStrict = VINF_SUCCESS;
15225	}
15226
15227	return rcStrict;
15228	}
15229
15230
15231	/**
15232	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
15233	* VM-exit.
15234	*/
15235	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15236	{
15237	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15238	return VINF_EM_RESET;
15239	}
15240
15241
15242	/**
15243	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15244	*/
15245	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15246	{
15247	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15248
15249	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15250	AssertRCReturn(rc, rc);
15251
15252	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
15253	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
15254	rc = VINF_SUCCESS;
15255	else
15256	rc = VINF_EM_HALT;
15257
15258	if (rc != VINF_SUCCESS)
15259	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
15260	return rc;
15261	}
15262
15263
15264	/**
15265	* VM-exit handler for instructions that result in a \#UD exception delivered to
15266	* the guest.
15267	*/
15268	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15269	{
15270	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15271	hmR0VmxSetPendingXcptUD(pVCpu);
15272	return VINF_SUCCESS;
15273	}
15274
15275
15276	/**
15277	* VM-exit handler for expiry of the VMX-preemption timer.
15278	*/
15279	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15280	{
15281	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15282
15283	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
15284	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15285
15286	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
15287	PVM pVM = pVCpu->CTX_SUFF(pVM);
15288	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
15289	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
15290	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
15291	}
15292
15293
15294	/**
15295	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
15296	*/
15297	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15298	{
15299	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15300
15301	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15302	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15303	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
15304	AssertRCReturn(rc, rc);
15305
15306	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
15307	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15308	: HM_CHANGED_RAISED_XCPT_MASK);
15309
15310	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15311	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
15312
15313	return rcStrict;
15314	}
15315
15316
15317	/**
15318	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
15319	*/
15320	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15321	{
15322	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15323	/** @todo Use VM-exit instruction information. */
15324	return VERR_EM_INTERPRETER;
15325	}
15326
15327
15328	/**
15329	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
15330	* VM-exit.
15331	*/
15332	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15333	{
15334	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15335	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15336	AssertRCReturn(rc, rc);
15337
15338	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
15339	if (RT_FAILURE(rc))
15340	return rc;
15341
15342	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
15343	NOREF(uInvalidReason);
15344
15345	#ifdef VBOX_STRICT
15346	uint32_t fIntrState;
15347	RTHCUINTREG uHCReg;
15348	uint64_t u64Val;
15349	uint32_t u32Val;
15350	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
15351	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
15352	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
15353	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
15354	AssertRCReturn(rc, rc);
15355
15356	Log4(("uInvalidReason %u\n", uInvalidReason));
15357	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
15358	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
15359	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
15360	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
15361
15362	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
15363	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
15364	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
15365	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
15366	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
15367	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15368	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
15369	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
15370	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
15371	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15372	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
15373	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
15374
15375	hmR0DumpRegs(pVCpu);
15376	#endif
15377
15378	return VERR_VMX_INVALID_GUEST_STATE;
15379	}
15380
15381	/**
15382	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
15383	*/
15384	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15385	{
15386	/*
15387	* Cummulative notes of all recognized but unexpected VM-exits.
15388	*
15389	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
15390	* nested-paging is used.
15391	*
15392	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
15393	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
15394	* this function (and thereby stop VM execution) for handling such instructions.
15395	*
15396	*
15397	* VMX_EXIT_INIT_SIGNAL:
15398	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15399	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15400	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15401	*
15402	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15403	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15404	* See Intel spec. "23.8 Restrictions on VMX operation".
15405	*
15406	* VMX_EXIT_SIPI:
15407	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15408	* activity state is used. We don't make use of it as our guests don't have direct
15409	* access to the host local APIC.
15410	*
15411	* See Intel spec. 25.3 "Other Causes of VM-exits".
15412	*
15413	* VMX_EXIT_IO_SMI:
15414	* VMX_EXIT_SMI:
15415	* This can only happen if we support dual-monitor treatment of SMI, which can be
15416	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15417	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15418	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15419	*
15420	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15421	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15422	*
15423	* VMX_EXIT_ERR_MSR_LOAD:
15424	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15425	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15426	* execution.
15427	*
15428	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15429	*
15430	* VMX_EXIT_ERR_MACHINE_CHECK:
15431	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15432	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15433	* #MC exception abort class exception is raised. We thus cannot assume a
15434	* reasonable chance of continuing any sort of execution and we bail.
15435	*
15436	* See Intel spec. 15.1 "Machine-check Architecture".
15437	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15438	*
15439	* VMX_EXIT_PML_FULL:
15440	* VMX_EXIT_VIRTUALIZED_EOI:
15441	* VMX_EXIT_APIC_WRITE:
15442	* We do not currently support any of these features and thus they are all unexpected
15443	* VM-exits.
15444	*
15445	* VMX_EXIT_GDTR_IDTR_ACCESS:
15446	* VMX_EXIT_LDTR_TR_ACCESS:
15447	* VMX_EXIT_RDRAND:
15448	* VMX_EXIT_RSM:
15449	* VMX_EXIT_VMFUNC:
15450	* VMX_EXIT_ENCLS:
15451	* VMX_EXIT_RDSEED:
15452	* VMX_EXIT_XSAVES:
15453	* VMX_EXIT_XRSTORS:
15454	* VMX_EXIT_UMWAIT:
15455	* VMX_EXIT_TPAUSE:
15456	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15457	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15458	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15459	*
15460	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15461	*/
15462	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15463	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15464	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15465	}
15466
15467
15468	/**
15469	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15470	*/
15471	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15472	{
15473	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15474
15475	/** @todo Optimize this: We currently drag in in the whole MSR state
15476	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15477	* MSRs required. That would require changes to IEM and possibly CPUM too.
15478	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15479	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15480	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15481	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15482	switch (idMsr)
15483	{
15484	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15485	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15486	}
15487
15488	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15489	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15490	AssertRCReturn(rc, rc);
15491
15492	Log4Func(("ecx=%#RX32\n", idMsr));
15493
15494	#ifdef VBOX_STRICT
15495	Assert(!pVmxTransient->fIsNestedGuest);
15496	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15497	{
15498	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15499	&& idMsr != MSR_K6_EFER)
15500	{
15501	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15502	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15503	}
15504	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15505	{
15506	Assert(pVmcsInfo->pvMsrBitmap);
15507	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15508	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15509	{
15510	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15511	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15512	}
15513	}
15514	}
15515	#endif
15516
15517	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15519	if (rcStrict == VINF_SUCCESS)
15520	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15521	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15522	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15523	{
15524	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15525	rcStrict = VINF_SUCCESS;
15526	}
15527	else
15528	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15529
15530	return rcStrict;
15531	}
15532
15533
15534	/**
15535	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15536	*/
15537	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15538	{
15539	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15540
15541	/** @todo Optimize this: We currently drag in in the whole MSR state
15542	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15543	* MSRs required. That would require changes to IEM and possibly CPUM too.
15544	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15545	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15546	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15547
15548	/*
15549	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15550	* Although we don't need to fetch the base as it will be overwritten shortly, while
15551	* loading guest-state we would also load the entire segment register including limit
15552	* and attributes and thus we need to load them here.
15553	*/
15554	switch (idMsr)
15555	{
15556	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15557	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15558	}
15559
15560	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15561	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15562	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15563	AssertRCReturn(rc, rc);
15564
15565	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15566
15567	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15569
15570	if (rcStrict == VINF_SUCCESS)
15571	{
15572	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15573
15574	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15575	if ( idMsr == MSR_IA32_APICBASE
15576	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15577	&& idMsr <= MSR_IA32_X2APIC_END))
15578	{
15579	/*
15580	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15581	* When full APIC register virtualization is implemented we'll have to make
15582	* sure APIC state is saved from the VMCS before IEM changes it.
15583	*/
15584	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15585	}
15586	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15587	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15588	else if (idMsr == MSR_K6_EFER)
15589	{
15590	/*
15591	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15592	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15593	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15594	*/
15595	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15596	}
15597
15598	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15599	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15600	{
15601	switch (idMsr)
15602	{
15603	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15604	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15605	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15606	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15607	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15608	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15609	default:
15610	{
15611	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15612	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15613	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15614	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15615	break;
15616	}
15617	}
15618	}
15619	#ifdef VBOX_STRICT
15620	else
15621	{
15622	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15623	switch (idMsr)
15624	{
15625	case MSR_IA32_SYSENTER_CS:
15626	case MSR_IA32_SYSENTER_EIP:
15627	case MSR_IA32_SYSENTER_ESP:
15628	case MSR_K8_FS_BASE:
15629	case MSR_K8_GS_BASE:
15630	{
15631	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15632	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15633	}
15634
15635	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15636	default:
15637	{
15638	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15639	{
15640	/* EFER MSR writes are always intercepted. */
15641	if (idMsr != MSR_K6_EFER)
15642	{
15643	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15644	idMsr));
15645	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15646	}
15647	}
15648
15649	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15650	{
15651	Assert(pVmcsInfo->pvMsrBitmap);
15652	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15653	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15654	{
15655	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15656	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15657	}
15658	}
15659	break;
15660	}
15661	}
15662	}
15663	#endif /* VBOX_STRICT */
15664	}
15665	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15666	{
15667	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15668	rcStrict = VINF_SUCCESS;
15669	}
15670	else
15671	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15672
15673	return rcStrict;
15674	}
15675
15676
15677	/**
15678	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15679	*/
15680	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15681	{
15682	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15683
15684	/** @todo The guest has likely hit a contended spinlock. We might want to
15685	* poke a schedule different guest VCPU. */
15686	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15687	if (RT_SUCCESS(rc))
15688	return VINF_EM_RAW_INTERRUPT;
15689
15690	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15691	return rc;
15692	}
15693
15694
15695	/**
15696	* VM-exit handler for when the TPR value is lowered below the specified
15697	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15698	*/
15699	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15700	{
15701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15702	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15703
15704	/*
15705	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15706	* We'll re-evaluate pending interrupts and inject them before the next VM
15707	* entry so we can just continue execution here.
15708	*/
15709	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15710	return VINF_SUCCESS;
15711	}
15712
15713
15714	/**
15715	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15716	* VM-exit.
15717	*
15718	* @retval VINF_SUCCESS when guest execution can continue.
15719	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15720	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15721	* incompatible guest state for VMX execution (real-on-v86 case).
15722	*/
15723	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15724	{
15725	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15726	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15727
15728	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15729	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15730	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15731	AssertRCReturn(rc, rc);
15732
15733	VBOXSTRICTRC rcStrict;
15734	PVM pVM = pVCpu->CTX_SUFF(pVM);
15735	uint64_t const uExitQual = pVmxTransient->uExitQual;
15736	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15737	switch (uAccessType)
15738	{
15739	/*
15740	* MOV to CRx.
15741	*/
15742	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15743	{
15744	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15745	AssertRCReturn(rc, rc);
15746
15747	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15748	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15749	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15750	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15751
15752	/*
15753	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15754	* - When nested paging isn't used.
15755	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15756	* - We are executing in the VM debug loop.
15757	*/
15758	Assert( iCrReg != 3
15759	\|\| !pVM->hm.s.fNestedPaging
15760	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15761	\|\| pVCpu->hm.s.fUsingDebugLoop);
15762
15763	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15764	Assert( iCrReg != 8
15765	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15766
15767	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15768	AssertMsg( rcStrict == VINF_SUCCESS
15769	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15770
15771	/*
15772	* This is a kludge for handling switches back to real mode when we try to use
15773	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15774	* deal with special selector values, so we have to return to ring-3 and run
15775	* there till the selector values are V86 mode compatible.
15776	*
15777	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15778	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15779	* this function.
15780	*/
15781	if ( iCrReg == 0
15782	&& rcStrict == VINF_SUCCESS
15783	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15784	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15785	&& (uOldCr0 & X86_CR0_PE)
15786	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15787	{
15788	/** @todo Check selectors rather than returning all the time. */
15789	Assert(!pVmxTransient->fIsNestedGuest);
15790	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15791	rcStrict = VINF_EM_RESCHEDULE_REM;
15792	}
15793	break;
15794	}
15795
15796	/*
15797	* MOV from CRx.
15798	*/
15799	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15800	{
15801	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15802	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15803
15804	/*
15805	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15806	* - When nested paging isn't used.
15807	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15808	* - We are executing in the VM debug loop.
15809	*/
15810	Assert( iCrReg != 3
15811	\|\| !pVM->hm.s.fNestedPaging
15812	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15813	\|\| pVCpu->hm.s.fUsingDebugLoop);
15814
15815	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15816	Assert( iCrReg != 8
15817	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15818
15819	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15820	break;
15821	}
15822
15823	/*
15824	* CLTS (Clear Task-Switch Flag in CR0).
15825	*/
15826	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15827	{
15828	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15829	break;
15830	}
15831
15832	/*
15833	* LMSW (Load Machine-Status Word into CR0).
15834	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15835	*/
15836	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15837	{
15838	RTGCPTR GCPtrEffDst;
15839	uint8_t const cbInstr = pVmxTransient->cbInstr;
15840	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15841	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15842	if (fMemOperand)
15843	{
15844	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15845	AssertRCReturn(rc, rc);
15846	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15847	}
15848	else
15849	GCPtrEffDst = NIL_RTGCPTR;
15850	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15851	break;
15852	}
15853
15854	default:
15855	{
15856	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15857	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15858	}
15859	}
15860
15861	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15862	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15863	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15864
15865	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15866	NOREF(pVM);
15867	return rcStrict;
15868	}
15869
15870
15871	/**
15872	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15873	* VM-exit.
15874	*/
15875	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15876	{
15877	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15878	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15879
15880	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15881	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15882	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15883	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15884	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15885	\| CPUMCTX_EXTRN_EFER);
15886	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15887	AssertRCReturn(rc, rc);
15888
15889	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15890	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15891	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15892	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15893	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15894	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15895	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15896	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15897
15898	/*
15899	* Update exit history to see if this exit can be optimized.
15900	*/
15901	VBOXSTRICTRC rcStrict;
15902	PCEMEXITREC pExitRec = NULL;
15903	if ( !fGstStepping
15904	&& !fDbgStepping)
15905	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15906	!fIOString
15907	? !fIOWrite
15908	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15909	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15910	: !fIOWrite
15911	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15912	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15913	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15914	if (!pExitRec)
15915	{
15916	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15917	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15918
15919	uint32_t const cbValue = s_aIOSizes[uIOSize];
15920	uint32_t const cbInstr = pVmxTransient->cbInstr;
15921	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15922	PVM pVM = pVCpu->CTX_SUFF(pVM);
15923	if (fIOString)
15924	{
15925	/*
15926	* INS/OUTS - I/O String instruction.
15927	*
15928	* Use instruction-information if available, otherwise fall back on
15929	* interpreting the instruction.
15930	*/
15931	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15932	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15933	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15934	if (fInsOutsInfo)
15935	{
15936	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15937	AssertRCReturn(rc2, rc2);
15938	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15939	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15940	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15941	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15942	if (fIOWrite)
15943	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15944	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15945	else
15946	{
15947	/*
15948	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15949	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15950	* See Intel Instruction spec. for "INS".
15951	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15952	*/
15953	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15954	}
15955	}
15956	else
15957	rcStrict = IEMExecOne(pVCpu);
15958
15959	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15960	fUpdateRipAlready = true;
15961	}
15962	else
15963	{
15964	/*
15965	* IN/OUT - I/O instruction.
15966	*/
15967	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15968	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15969	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15970	if (fIOWrite)
15971	{
15972	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15973	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15974	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15975	&& !pCtx->eflags.Bits.u1TF)
15976	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15977	}
15978	else
15979	{
15980	uint32_t u32Result = 0;
15981	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15982	if (IOM_SUCCESS(rcStrict))
15983	{
15984	/* Save result of I/O IN instr. in AL/AX/EAX. */
15985	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15986	}
15987	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15988	&& !pCtx->eflags.Bits.u1TF)
15989	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15990	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15991	}
15992	}
15993
15994	if (IOM_SUCCESS(rcStrict))
15995	{
15996	if (!fUpdateRipAlready)
15997	{
15998	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15999	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
16000	}
16001
16002	/*
16003	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
16004	* while booting Fedora 17 64-bit guest.
16005	*
16006	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
16007	*/
16008	if (fIOString)
16009	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
16010
16011	/*
16012	* If any I/O breakpoints are armed, we need to check if one triggered
16013	* and take appropriate action.
16014	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
16015	*/
16016	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
16017	AssertRCReturn(rc, rc);
16018
16019	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
16020	* execution engines about whether hyper BPs and such are pending. */
16021	uint32_t const uDr7 = pCtx->dr[7];
16022	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
16023	&& X86_DR7_ANY_RW_IO(uDr7)
16024	&& (pCtx->cr4 & X86_CR4_DE))
16025	\|\| DBGFBpIsHwIoArmed(pVM)))
16026	{
16027	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
16028
16029	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
16030	VMMRZCallRing3Disable(pVCpu);
16031	HM_DISABLE_PREEMPT(pVCpu);
16032
16033	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
16034
16035	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
16036	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
16037	{
16038	/* Raise #DB. */
16039	if (fIsGuestDbgActive)
16040	ASMSetDR6(pCtx->dr[6]);
16041	if (pCtx->dr[7] != uDr7)
16042	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
16043
16044	hmR0VmxSetPendingXcptDB(pVCpu);
16045	}
16046	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
16047	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
16048	else if ( rcStrict2 != VINF_SUCCESS
16049	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
16050	rcStrict = rcStrict2;
16051	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
16052
16053	HM_RESTORE_PREEMPT();
16054	VMMRZCallRing3Enable(pVCpu);
16055	}
16056	}
16057
16058	#ifdef VBOX_STRICT
16059	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
16060	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
16061	Assert(!fIOWrite);
16062	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
16063	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
16064	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
16065	Assert(fIOWrite);
16066	else
16067	{
16068	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
16069	* statuses, that the VMM device and some others may return. See
16070	* IOM_SUCCESS() for guidance. */
16071	AssertMsg( RT_FAILURE(rcStrict)
16072	\|\| rcStrict == VINF_SUCCESS
16073	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
16074	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
16075	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
16076	\|\| rcStrict == VINF_EM_RAW_TO_R3
16077	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16078	# endif
16079	}
16080	#endif
16081	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
16082	}
16083	else
16084	{
16085	/*
16086	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16087	*/
16088	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16089	AssertRCReturn(rc2, rc2);
16090	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
16091	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
16092	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
16093	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16094	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
16095	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
16096
16097	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16098	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16099
16100	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16101	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16102	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16103	}
16104	return rcStrict;
16105	}
16106
16107
16108	/**
16109	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
16110	* VM-exit.
16111	*/
16112	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16113	{
16114	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16115
16116	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
16117	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16118	AssertRCReturn(rc, rc);
16119	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
16120	{
16121	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16122	AssertRCReturn(rc, rc);
16123	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16124	{
16125	uint32_t uErrCode;
16126	RTGCUINTPTR GCPtrFaultAddress;
16127	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
16128	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
16129	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
16130	if (fErrorCodeValid)
16131	{
16132	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16133	AssertRCReturn(rc, rc);
16134	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
16135	}
16136	else
16137	uErrCode = 0;
16138
16139	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
16140	&& uVector == X86_XCPT_PF)
16141	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
16142	else
16143	GCPtrFaultAddress = 0;
16144
16145	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16146	AssertRCReturn(rc, rc);
16147
16148	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
16149	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
16150
16151	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
16152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16153	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16154	}
16155	}
16156
16157	/* Fall back to the interpreter to emulate the task-switch. */
16158	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16159	return VERR_EM_INTERPRETER;
16160	}
16161
16162
16163	/**
16164	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
16165	*/
16166	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16167	{
16168	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16169
16170	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16171	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
16172	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16173	AssertRCReturn(rc, rc);
16174	return VINF_EM_DBG_STEPPED;
16175	}
16176
16177
16178	/**
16179	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
16180	*/
16181	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16182	{
16183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
16185
16186	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16187	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16188	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16189	{
16190	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
16191	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16192	{
16193	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
16194	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16195	}
16196	}
16197	else
16198	{
16199	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16200	rcStrict1 = VINF_SUCCESS;
16201	return rcStrict1;
16202	}
16203
16204	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
16205	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16206	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16207	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16208	AssertRCReturn(rc, rc);
16209
16210	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
16211	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
16212	VBOXSTRICTRC rcStrict2;
16213	switch (uAccessType)
16214	{
16215	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
16216	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
16217	{
16218	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
16219	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
16220	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
16221
16222	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
16223	GCPhys &= PAGE_BASE_GC_MASK;
16224	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
16225	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
16226	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
16227
16228	PVM pVM = pVCpu->CTX_SUFF(pVM);
16229	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16230	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
16231	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
16232	CPUMCTX2CORE(pCtx), GCPhys);
16233	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16234	if ( rcStrict2 == VINF_SUCCESS
16235	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16236	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16237	{
16238	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16239	\| HM_CHANGED_GUEST_APIC_TPR);
16240	rcStrict2 = VINF_SUCCESS;
16241	}
16242	break;
16243	}
16244
16245	default:
16246	{
16247	Log4Func(("uAccessType=%#x\n", uAccessType));
16248	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
16249	break;
16250	}
16251	}
16252
16253	if (rcStrict2 != VINF_SUCCESS)
16254	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
16255	return rcStrict2;
16256	}
16257
16258
16259	/**
16260	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
16261	* VM-exit.
16262	*/
16263	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16264	{
16265	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16266
16267	/* We should -not- get this VM-exit if the guest's debug registers were active. */
16268	if (pVmxTransient->fWasGuestDebugStateActive)
16269	{
16270	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
16271	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
16272	}
16273
16274	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16275	if ( !pVCpu->hm.s.fSingleInstruction
16276	&& !pVmxTransient->fWasHyperDebugStateActive)
16277	{
16278	Assert(!DBGFIsStepping(pVCpu));
16279	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
16280
16281	/* Don't intercept MOV DRx any more. */
16282	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
16283	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16284	AssertRCReturn(rc, rc);
16285
16286	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
16287	VMMRZCallRing3Disable(pVCpu);
16288	HM_DISABLE_PREEMPT(pVCpu);
16289
16290	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
16291	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
16292	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
16293
16294	HM_RESTORE_PREEMPT();
16295	VMMRZCallRing3Enable(pVCpu);
16296
16297	#ifdef VBOX_WITH_STATISTICS
16298	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16299	AssertRCReturn(rc, rc);
16300	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16302	else
16303	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16304	#endif
16305	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
16306	return VINF_SUCCESS;
16307	}
16308
16309	/*
16310	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
16311	* The EFER MSR is always up-to-date.
16312	* Update the segment registers and DR7 from the CPU.
16313	*/
16314	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16315	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16316	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
16317	AssertRCReturn(rc, rc);
16318	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
16319
16320	PVM pVM = pVCpu->CTX_SUFF(pVM);
16321	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16322	{
16323	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16324	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
16325	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
16326	if (RT_SUCCESS(rc))
16327	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
16328	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16329	}
16330	else
16331	{
16332	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16333	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
16334	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
16335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16336	}
16337
16338	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
16339	if (RT_SUCCESS(rc))
16340	{
16341	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
16342	AssertRCReturn(rc2, rc2);
16343	return VINF_SUCCESS;
16344	}
16345	return rc;
16346	}
16347
16348
16349	/**
16350	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
16351	* Conditional VM-exit.
16352	*/
16353	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16354	{
16355	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16356	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16357
16358	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16359	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16360	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16361	{
16362	/*
16363	* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation. Otherwise,
16364	* injecting the original event would most likely cause the same EPT misconfig VM-exit again.
16365	*/
16366	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16367	{
16368	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
16369	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16370	}
16371	}
16372	else
16373	{
16374	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16375	rcStrict1 = VINF_SUCCESS;
16376	return rcStrict1;
16377	}
16378
16379	/*
16380	* Get sufficent state and update the exit history entry.
16381	*/
16382	RTGCPHYS GCPhys;
16383	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16384	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16385	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16386	AssertRCReturn(rc, rc);
16387
16388	VBOXSTRICTRC rcStrict;
16389	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16390	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16391	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16392	if (!pExitRec)
16393	{
16394	/*
16395	* If we succeed, resume guest execution.
16396	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16397	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16398	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16399	* weird case. See @bugref{6043}.
16400	*/
16401	PVM pVM = pVCpu->CTX_SUFF(pVM);
16402	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16403	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16404	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16405	if ( rcStrict == VINF_SUCCESS
16406	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16407	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16408	{
16409	/* Successfully handled MMIO operation. */
16410	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16411	\| HM_CHANGED_GUEST_APIC_TPR);
16412	rcStrict = VINF_SUCCESS;
16413	}
16414	}
16415	else
16416	{
16417	/*
16418	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16419	*/
16420	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16421	AssertRCReturn(rc2, rc2);
16422
16423	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16424	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16425
16426	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16427	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16428
16429	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16430	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16431	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16432	}
16433	return rcStrict;
16434	}
16435
16436
16437	/**
16438	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16439	* VM-exit.
16440	*/
16441	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16442	{
16443	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16444	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16445
16446	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16447	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16448	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16449	{
16450	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
16451	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16452	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
16453	}
16454	else
16455	{
16456	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16457	rcStrict1 = VINF_SUCCESS;
16458	return rcStrict1;
16459	}
16460
16461	RTGCPHYS GCPhys;
16462	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16463	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16464	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16465	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16466	AssertRCReturn(rc, rc);
16467
16468	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
16469	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
16470
16471	RTGCUINT uErrorCode = 0;
16472	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16473	uErrorCode \|= X86_TRAP_PF_ID;
16474	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16475	uErrorCode \|= X86_TRAP_PF_RW;
16476	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16477	uErrorCode \|= X86_TRAP_PF_P;
16478
16479	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16480
16481
16482	/* Handle the pagefault trap for the nested shadow table. */
16483	PVM pVM = pVCpu->CTX_SUFF(pVM);
16484	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16485
16486	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
16487	pCtx->cs.Sel, pCtx->rip));
16488
16489	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16490	TRPMResetTrap(pVCpu);
16491
16492	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16493	if ( rcStrict2 == VINF_SUCCESS
16494	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16495	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16496	{
16497	/* Successfully synced our nested page tables. */
16498	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16499	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16500	return VINF_SUCCESS;
16501	}
16502
16503	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16504	return rcStrict2;
16505	}
16506
16507
16508	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16509	/**
16510	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16511	*/
16512	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16513	{
16514	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16515
16516	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16517	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16518	\| CPUMCTX_EXTRN_HWVIRT
16519	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16520	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16521	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16522	AssertRCReturn(rc, rc);
16523
16524	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16525
16526	VMXVEXITINFO ExitInfo;
16527	RT_ZERO(ExitInfo);
16528	ExitInfo.uReason = pVmxTransient->uExitReason;
16529	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16530	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16531	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16532	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16533
16534	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16535	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16536	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16537	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16538	{
16539	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16540	rcStrict = VINF_SUCCESS;
16541	}
16542	return rcStrict;
16543	}
16544
16545
16546	/**
16547	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16548	*/
16549	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16550	{
16551	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16552
16553	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16554	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16555	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16556	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16557	AssertRCReturn(rc, rc);
16558
16559	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16560
16561	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16562	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16563	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16564	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16565	{
16566	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16567	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16568	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16569	}
16570	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16571	return rcStrict;
16572	}
16573
16574
16575	/**
16576	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16577	*/
16578	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16579	{
16580	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16581
16582	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16583	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16584	\| CPUMCTX_EXTRN_HWVIRT
16585	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16586	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16587	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16588	AssertRCReturn(rc, rc);
16589
16590	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16591
16592	VMXVEXITINFO ExitInfo;
16593	RT_ZERO(ExitInfo);
16594	ExitInfo.uReason = pVmxTransient->uExitReason;
16595	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16596	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16597	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16598	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16599
16600	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16601	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16602	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16603	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16604	{
16605	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16606	rcStrict = VINF_SUCCESS;
16607	}
16608	return rcStrict;
16609	}
16610
16611
16612	/**
16613	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16614	*/
16615	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16616	{
16617	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16618
16619	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16620	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16621	\| CPUMCTX_EXTRN_HWVIRT
16622	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16623	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16624	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16625	AssertRCReturn(rc, rc);
16626
16627	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16628
16629	VMXVEXITINFO ExitInfo;
16630	RT_ZERO(ExitInfo);
16631	ExitInfo.uReason = pVmxTransient->uExitReason;
16632	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16633	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16634	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16635	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16636
16637	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16638	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16639	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16640	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16641	{
16642	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16643	rcStrict = VINF_SUCCESS;
16644	}
16645	return rcStrict;
16646	}
16647
16648
16649	/**
16650	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16651	*/
16652	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16653	{
16654	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16655
16656	/*
16657	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16658	* thus might not need to import the shadow VMCS state, it's safer just in case
16659	* code elsewhere dares look at unsynced VMCS fields.
16660	*/
16661	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16662	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16663	\| CPUMCTX_EXTRN_HWVIRT
16664	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16665	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16666	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16667	AssertRCReturn(rc, rc);
16668
16669	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16670
16671	VMXVEXITINFO ExitInfo;
16672	RT_ZERO(ExitInfo);
16673	ExitInfo.uReason = pVmxTransient->uExitReason;
16674	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16675	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16676	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16677	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16678	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16679
16680	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16681	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16682	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16683	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16684	{
16685	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16686	rcStrict = VINF_SUCCESS;
16687	}
16688	return rcStrict;
16689	}
16690
16691
16692	/**
16693	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16694	*/
16695	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16696	{
16697	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16698
16699	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16700	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16701	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16702	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16703	AssertRCReturn(rc, rc);
16704
16705	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16706
16707	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16708	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16709	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16710	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16711	{
16712	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16713	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16714	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16715	}
16716	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16717	return rcStrict;
16718	}
16719
16720
16721	/**
16722	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16723	*/
16724	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16725	{
16726	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16727
16728	/*
16729	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16730	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16731	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16732	*/
16733	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16734	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16735	\| CPUMCTX_EXTRN_HWVIRT
16736	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16737	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16738	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16739	AssertRCReturn(rc, rc);
16740
16741	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16742
16743	VMXVEXITINFO ExitInfo;
16744	RT_ZERO(ExitInfo);
16745	ExitInfo.uReason = pVmxTransient->uExitReason;
16746	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16747	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16748	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16749	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16750	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16751
16752	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16753	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16754	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16755	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16756	{
16757	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16758	rcStrict = VINF_SUCCESS;
16759	}
16760	return rcStrict;
16761	}
16762
16763
16764	/**
16765	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16766	*/
16767	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16768	{
16769	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16770
16771	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16772	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16773	\| CPUMCTX_EXTRN_HWVIRT
16774	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16775	AssertRCReturn(rc, rc);
16776
16777	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16778
16779	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16780	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16781	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16782	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16783	{
16784	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16785	rcStrict = VINF_SUCCESS;
16786	}
16787	return rcStrict;
16788	}
16789
16790
16791	/**
16792	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16793	*/
16794	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16795	{
16796	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16797
16798	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16799	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16800	\| CPUMCTX_EXTRN_HWVIRT
16801	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16802	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16803	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16804	AssertRCReturn(rc, rc);
16805
16806	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16807
16808	VMXVEXITINFO ExitInfo;
16809	RT_ZERO(ExitInfo);
16810	ExitInfo.uReason = pVmxTransient->uExitReason;
16811	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16812	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16813	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16814	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16815
16816	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16817	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16818	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16819	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16820	{
16821	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16822	rcStrict = VINF_SUCCESS;
16823	}
16824	return rcStrict;
16825	}
16826
16827
16828	/**
16829	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16830	*/
16831	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16832	{
16833	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16834
16835	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16836	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16837	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16838	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16839	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16840	AssertRCReturn(rc, rc);
16841
16842	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16843
16844	VMXVEXITINFO ExitInfo;
16845	RT_ZERO(ExitInfo);
16846	ExitInfo.uReason = pVmxTransient->uExitReason;
16847	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16848	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16849	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16850	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16851
16852	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16853	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16854	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16855	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16856	{
16857	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16858	rcStrict = VINF_SUCCESS;
16859	}
16860	return rcStrict;
16861	}
16862	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16863	/** @} */
16864
16865
16866	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16867	/** @name Nested-guest VM-exit handlers.
16868	* @{
16869	*/
16870	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16871	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16872	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16873
16874	/**
16875	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16876	* Conditional VM-exit.
16877	*/
16878	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16879	{
16880	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16881
16882	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16883	AssertRCReturn(rc, rc);
16884
16885	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16886	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16887	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16888
16889	/*
16890	* Make sure not to use stale/previous VM-exit instruction length since we read the
16891	* instruction length from the VMCS below only for software exceptions and privileged
16892	* software exceptions but we pass it for all exception VM-exits below.
16893	*/
16894	pVmxTransient->cbInstr = 0;
16895
16896	switch (uExtIntType)
16897	{
16898	/*
16899	* Physical NMIs:
16900	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16901	*/
16902	case VMX_EXIT_INT_INFO_TYPE_NMI:
16903	return hmR0VmxExitHostNmi(pVCpu);
16904
16905	/*
16906	* Hardware exceptions,
16907	* Software exceptions,
16908	* Privileged software exceptions:
16909	* Figure out if the exception must be delivered to the guest or the nested-guest.
16910	*
16911	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16912	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16913	* length.
16914	*/
16915	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16916	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16917	{
16918	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16919	AssertRCReturn(rc, rc);
16920	RT_FALL_THRU();
16921	}
16922	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16923	{
16924	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16925	AssertRCReturn(rc, rc);
16926
16927	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
16928	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
16929	pVmxTransient->uExitIntErrorCode);
16930	if (fIntercept)
16931	{
16932	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16933	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16934	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16935	AssertRCReturn(rc, rc);
16936
16937	VMXVEXITINFO ExitInfo;
16938	RT_ZERO(ExitInfo);
16939	ExitInfo.uReason = pVmxTransient->uExitReason;
16940	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16941	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16942
16943	VMXVEXITEVENTINFO ExitEventInfo;
16944	RT_ZERO(ExitEventInfo);
16945	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16946	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16947	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16948	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16949
16950	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16951	}
16952
16953	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
16954	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16955
16956	/* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
16957	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
16958	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16959	return VINF_SUCCESS;
16960	}
16961
16962	/*
16963	* Software interrupts:
16964	* VM-exits cannot be caused by software interrupts.
16965	*
16966	* External interrupts:
16967	* This should only happen when "acknowledge external interrupts on VM-exit"
16968	* control is set. However, we never set this when executing a guest or
16969	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16970	* the guest.
16971	*/
16972	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16973	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16974	default:
16975	{
16976	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16977	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16978	}
16979	}
16980	}
16981
16982
16983	/**
16984	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16985	* Unconditional VM-exit.
16986	*/
16987	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16988	{
16989	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16990	return IEMExecVmxVmexitTripleFault(pVCpu);
16991	}
16992
16993
16994	/**
16995	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16996	*/
16997	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16998	{
16999	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17000
17001	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
17002	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17003	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
17004	}
17005
17006
17007	/**
17008	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
17009	*/
17010	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17011	{
17012	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17013
17014	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
17015	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17016	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
17017	}
17018
17019
17020	/**
17021	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
17022	* Unconditional VM-exit.
17023	*/
17024	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17025	{
17026	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17027
17028	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17029	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17030	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17031	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17032	AssertRCReturn(rc, rc);
17033
17034	VMXVEXITINFO ExitInfo;
17035	RT_ZERO(ExitInfo);
17036	ExitInfo.uReason = pVmxTransient->uExitReason;
17037	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17038	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17039
17040	VMXVEXITEVENTINFO ExitEventInfo;
17041	RT_ZERO(ExitEventInfo);
17042	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17043	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17044	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
17045	}
17046
17047
17048	/**
17049	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
17050	*/
17051	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17052	{
17053	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17054
17055	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
17056	{
17057	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17058	AssertRCReturn(rc, rc);
17059	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17060	}
17061	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
17062	}
17063
17064
17065	/**
17066	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
17067	*/
17068	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17069	{
17070	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17071
17072	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17073	{
17074	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17075	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17076	AssertRCReturn(rc, rc);
17077
17078	VMXVEXITINFO ExitInfo;
17079	RT_ZERO(ExitInfo);
17080	ExitInfo.uReason = pVmxTransient->uExitReason;
17081	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17082	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17083	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17084	}
17085	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
17086	}
17087
17088
17089	/**
17090	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
17091	*/
17092	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17093	{
17094	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17095
17096	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
17097	{
17098	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17099	AssertRCReturn(rc, rc);
17100	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17101	}
17102	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
17103	}
17104
17105
17106	/**
17107	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
17108	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
17109	*/
17110	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17111	{
17112	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17113
17114	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
17115	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
17116
17117	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17118	AssertRCReturn(rc, rc);
17119
17120	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
17121	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17122	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17123
17124	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
17125	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
17126	u64VmcsField &= UINT64_C(0xffffffff);
17127
17128	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
17129	{
17130	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17131	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17132	AssertRCReturn(rc, rc);
17133
17134	VMXVEXITINFO ExitInfo;
17135	RT_ZERO(ExitInfo);
17136	ExitInfo.uReason = pVmxTransient->uExitReason;
17137	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17138	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17139	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17140	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17141	}
17142
17143	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
17144	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
17145	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
17146	}
17147
17148
17149	/**
17150	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
17151	*/
17152	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17153	{
17154	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17155
17156	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17157	{
17158	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17159	AssertRCReturn(rc, rc);
17160	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17161	}
17162
17163	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
17164	}
17165
17166
17167	/**
17168	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
17169	* Conditional VM-exit.
17170	*/
17171	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17172	{
17173	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17174
17175	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17176	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17177	AssertRCReturn(rc, rc);
17178
17179	VBOXSTRICTRC rcStrict;
17180	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
17181	switch (uAccessType)
17182	{
17183	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
17184	{
17185	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17186	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17187	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17188	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17189
17190	bool fIntercept;
17191	switch (iCrReg)
17192	{
17193	case 0:
17194	case 4:
17195	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
17196	break;
17197
17198	case 3:
17199	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
17200	break;
17201
17202	case 8:
17203	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
17204	break;
17205
17206	default:
17207	fIntercept = false;
17208	break;
17209	}
17210	if (fIntercept)
17211	{
17212	VMXVEXITINFO ExitInfo;
17213	RT_ZERO(ExitInfo);
17214	ExitInfo.uReason = pVmxTransient->uExitReason;
17215	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17216	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17217	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17218	}
17219	else
17220	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17221	break;
17222	}
17223
17224	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
17225	{
17226	/*
17227	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
17228	* CR2 reads do not cause a VM-exit.
17229	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
17230	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
17231	*/
17232	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17233	if ( iCrReg == 3
17234	\|\| iCrReg == 8)
17235	{
17236	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
17237	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
17238	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
17239	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
17240	{
17241	VMXVEXITINFO ExitInfo;
17242	RT_ZERO(ExitInfo);
17243	ExitInfo.uReason = pVmxTransient->uExitReason;
17244	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17245	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17246	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17247	}
17248	else
17249	{
17250	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17251	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17252	}
17253	}
17254	else
17255	{
17256	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
17257	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
17258	}
17259	break;
17260	}
17261
17262	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
17263	{
17264	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
17265	Assert(pVmcsNstGst);
17266	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
17267	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
17268	if ( (uGstHostMask & X86_CR0_TS)
17269	&& (uReadShadow & X86_CR0_TS))
17270	{
17271	VMXVEXITINFO ExitInfo;
17272	RT_ZERO(ExitInfo);
17273	ExitInfo.uReason = pVmxTransient->uExitReason;
17274	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17275	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17276	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17277	}
17278	else
17279	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
17280	break;
17281	}
17282
17283	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
17284	{
17285	RTGCPTR GCPtrEffDst;
17286	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
17287	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
17288	if (fMemOperand)
17289	{
17290	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17291	AssertRCReturn(rc, rc);
17292	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
17293	}
17294	else
17295	GCPtrEffDst = NIL_RTGCPTR;
17296
17297	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
17298	{
17299	VMXVEXITINFO ExitInfo;
17300	RT_ZERO(ExitInfo);
17301	ExitInfo.uReason = pVmxTransient->uExitReason;
17302	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17303	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
17304	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17305	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17306	}
17307	else
17308	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
17309	break;
17310	}
17311
17312	default:
17313	{
17314	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
17315	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
17316	}
17317	}
17318
17319	if (rcStrict == VINF_IEM_RAISED_XCPT)
17320	{
17321	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
17322	rcStrict = VINF_SUCCESS;
17323	}
17324	return rcStrict;
17325	}
17326
17327
17328	/**
17329	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
17330	* Conditional VM-exit.
17331	*/
17332	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17333	{
17334	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17335
17336	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
17337	{
17338	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17339	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17340	AssertRCReturn(rc, rc);
17341
17342	VMXVEXITINFO ExitInfo;
17343	RT_ZERO(ExitInfo);
17344	ExitInfo.uReason = pVmxTransient->uExitReason;
17345	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17346	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17347	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17348	}
17349	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
17350	}
17351
17352
17353	/**
17354	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
17355	* Conditional VM-exit.
17356	*/
17357	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17358	{
17359	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17360
17361	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17362	AssertRCReturn(rc, rc);
17363
17364	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
17365	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
17366	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
17367
17368	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
17369	uint8_t const cbAccess = s_aIOSizes[uIOSize];
17370	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
17371	{
17372	/*
17373	* IN/OUT instruction:
17374	* - Provides VM-exit instruction length.
17375	*
17376	* INS/OUTS instruction:
17377	* - Provides VM-exit instruction length.
17378	* - Provides Guest-linear address.
17379	* - Optionally provides VM-exit instruction info (depends on CPU feature).
17380	*/
17381	PVM pVM = pVCpu->CTX_SUFF(pVM);
17382	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17383	AssertRCReturn(rc, rc);
17384
17385	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17386	pVmxTransient->ExitInstrInfo.u = 0;
17387	pVmxTransient->uGuestLinearAddr = 0;
17388
17389	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17390	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17391	if (fIOString)
17392	{
17393	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17394	if (fVmxInsOutsInfo)
17395	{
17396	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17397	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17398	}
17399	}
17400	AssertRCReturn(rc, rc);
17401
17402	VMXVEXITINFO ExitInfo;
17403	RT_ZERO(ExitInfo);
17404	ExitInfo.uReason = pVmxTransient->uExitReason;
17405	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17406	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17407	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17408	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17409	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17410	}
17411	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17412	}
17413
17414
17415	/**
17416	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17417	*/
17418	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17419	{
17420	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17421
17422	uint32_t fMsrpm;
17423	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17424	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17425	else
17426	fMsrpm = VMXMSRPM_EXIT_RD;
17427
17428	if (fMsrpm & VMXMSRPM_EXIT_RD)
17429	{
17430	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17431	AssertRCReturn(rc, rc);
17432	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17433	}
17434	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17435	}
17436
17437
17438	/**
17439	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17440	*/
17441	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17442	{
17443	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17444
17445	uint32_t fMsrpm;
17446	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17447	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17448	else
17449	fMsrpm = VMXMSRPM_EXIT_WR;
17450
17451	if (fMsrpm & VMXMSRPM_EXIT_WR)
17452	{
17453	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17454	AssertRCReturn(rc, rc);
17455	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17456	}
17457	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17458	}
17459
17460
17461	/**
17462	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17463	*/
17464	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17465	{
17466	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17467
17468	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17469	{
17470	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17471	AssertRCReturn(rc, rc);
17472	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17473	}
17474	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17475	}
17476
17477
17478	/**
17479	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17480	* VM-exit.
17481	*/
17482	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17483	{
17484	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17485
17486	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17487	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17488	}
17489
17490
17491	/**
17492	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17493	*/
17494	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17495	{
17496	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17497
17498	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17499	{
17500	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17501	AssertRCReturn(rc, rc);
17502	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17503	}
17504	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17505	}
17506
17507
17508	/**
17509	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17510	*/
17511	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17512	{
17513	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17514
17515	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17516	* PAUSE when executing a nested-guest? If it does not, we would not need
17517	* to check for the intercepts here. Just call VM-exit... */
17518
17519	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17520	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17521	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17522	{
17523	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17524	AssertRCReturn(rc, rc);
17525	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17526	}
17527	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17528	}
17529
17530
17531	/**
17532	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17533	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17534	*/
17535	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17536	{
17537	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17538
17539	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17540	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17541	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17542	}
17543
17544
17545	/**
17546	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17547	* VM-exit.
17548	*/
17549	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17550	{
17551	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17552
17553	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17554	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17555	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17556	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17557	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17558	AssertRCReturn(rc, rc);
17559
17560	VMXVEXITINFO ExitInfo;
17561	RT_ZERO(ExitInfo);
17562	ExitInfo.uReason = pVmxTransient->uExitReason;
17563	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17564	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17565
17566	VMXVEXITEVENTINFO ExitEventInfo;
17567	RT_ZERO(ExitEventInfo);
17568	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17569	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17570	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17571	}
17572
17573
17574	/**
17575	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17576	* Conditional VM-exit.
17577	*/
17578	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17579	{
17580	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17581
17582	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17583	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17584	AssertRCReturn(rc, rc);
17585
17586	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17587	}
17588
17589
17590	/**
17591	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17592	* Conditional VM-exit.
17593	*/
17594	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17595	{
17596	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17597
17598	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17599	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17600	AssertRCReturn(rc, rc);
17601
17602	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17603	}
17604
17605
17606	/**
17607	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17608	*/
17609	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17610	{
17611	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17612
17613	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17614	{
17615	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17616	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17617	AssertRCReturn(rc, rc);
17618	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17619	}
17620	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17621	}
17622
17623
17624	/**
17625	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17626	*/
17627	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17628	{
17629	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17630
17631	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17632	{
17633	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17634	AssertRCReturn(rc, rc);
17635	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17636	}
17637	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17638	}
17639
17640
17641	/**
17642	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17643	*/
17644	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17645	{
17646	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17647
17648	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17649	{
17650	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17651	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17652	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17653	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17654	AssertRCReturn(rc, rc);
17655
17656	VMXVEXITINFO ExitInfo;
17657	RT_ZERO(ExitInfo);
17658	ExitInfo.uReason = pVmxTransient->uExitReason;
17659	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17660	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17661	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17662	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17663	}
17664	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17665	}
17666
17667
17668	/**
17669	* Nested-guest VM-exit handler for invalid-guest state
17670	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17671	*/
17672	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17673	{
17674	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17675
17676	/*
17677	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17678	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17679	* Handle it like it's in an invalid guest state of the outer guest.
17680	*
17681	* When the fast path is implemented, this should be changed to cause the corresponding
17682	* nested-guest VM-exit.
17683	*/
17684	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17685	}
17686
17687
17688	/**
17689	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17690	* and only provide the instruction length.
17691	*
17692	* Unconditional VM-exit.
17693	*/
17694	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17695	{
17696	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17697
17698	#ifdef VBOX_STRICT
17699	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17700	switch (pVmxTransient->uExitReason)
17701	{
17702	case VMX_EXIT_ENCLS:
17703	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17704	break;
17705
17706	case VMX_EXIT_VMFUNC:
17707	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17708	break;
17709	}
17710	#endif
17711
17712	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17713	AssertRCReturn(rc, rc);
17714	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17715	}
17716
17717
17718	/**
17719	* Nested-guest VM-exit handler for instructions that provide instruction length as
17720	* well as more information.
17721	*
17722	* Unconditional VM-exit.
17723	*/
17724	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17725	{
17726	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17727
17728	#ifdef VBOX_STRICT
17729	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17730	switch (pVmxTransient->uExitReason)
17731	{
17732	case VMX_EXIT_GDTR_IDTR_ACCESS:
17733	case VMX_EXIT_LDTR_TR_ACCESS:
17734	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17735	break;
17736
17737	case VMX_EXIT_RDRAND:
17738	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17739	break;
17740
17741	case VMX_EXIT_RDSEED:
17742	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17743	break;
17744
17745	case VMX_EXIT_XSAVES:
17746	case VMX_EXIT_XRSTORS:
17747	/** @todo NSTVMX: Verify XSS-bitmap. */
17748	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17749	break;
17750
17751	case VMX_EXIT_UMWAIT:
17752	case VMX_EXIT_TPAUSE:
17753	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17754	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17755	break;
17756	}
17757	#endif
17758
17759	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17760	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17761	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17762	AssertRCReturn(rc, rc);
17763
17764	VMXVEXITINFO ExitInfo;
17765	RT_ZERO(ExitInfo);
17766	ExitInfo.uReason = pVmxTransient->uExitReason;
17767	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17768	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17769	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17770	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17771	}
17772
17773	/** @} */
17774	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17775

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79601

Download in other formats: