HMVMXR0.cpp@ 79637

Last change on this file since 79637 was 79637, checked in by vboxsync, 6 years ago
VMM/HM: Nested VMX: bugref:9180 Unify TRPM to VMX event conversion, let's not have 3 places doing it.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 730.3 KB

Line
1	/* $Id: HMVMXR0.cpp 79637 2019-07-09 09:08:37Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#include <VBox/vmm/em.h>
36	#include <VBox/vmm/gim.h>
37	#include <VBox/vmm/apic.h>
38	#ifdef VBOX_WITH_REM
39	# include <VBox/vmm/rem.h>
40	#endif
41	#include "HMInternal.h"
42	#include <VBox/vmm/vm.h>
43	#include <VBox/vmm/hmvmxinline.h>
44	#include "HMVMXR0.h"
45	#include "dtrace/VBoxVMM.h"
46
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
51	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
52	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
53	# define HMVMX_ALWAYS_TRAP_PF
54	# define HMVMX_ALWAYS_FLUSH_TLB
55	# define HMVMX_ALWAYS_SWAP_EFER
56	#endif
57
58
59	/*********************************************************************************************************************************
60	* Defined Constants And Macros *
61	*********************************************************************************************************************************/
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name HMVMX_READ_XXX
72	* Flags to skip redundant reads of some common VMCS fields that are not part of
73	* the guest-CPU or VCPU state but are needed while handling VM-exits.
74	*/
75	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
76	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
77	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
78	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
79	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
80	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
81	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
82	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
83	/** @} */
84
85	/**
86	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
87	* guest using hardware-assisted VMX.
88	*
89	* This excludes state like GPRs (other than RSP) which are always are
90	* swapped and restored across the world-switch and also registers like EFER,
91	* MSR which cannot be modified by the guest without causing a VM-exit.
92	*/
93	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
94	\| CPUMCTX_EXTRN_RFLAGS \
95	\| CPUMCTX_EXTRN_RSP \
96	\| CPUMCTX_EXTRN_SREG_MASK \
97	\| CPUMCTX_EXTRN_TABLE_MASK \
98	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
99	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
100	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
101	\| CPUMCTX_EXTRN_TSC_AUX \
102	\| CPUMCTX_EXTRN_OTHER_MSRS \
103	\| CPUMCTX_EXTRN_CR0 \
104	\| CPUMCTX_EXTRN_CR3 \
105	\| CPUMCTX_EXTRN_CR4 \
106	\| CPUMCTX_EXTRN_DR7 \
107	\| CPUMCTX_EXTRN_HWVIRT \
108	\| CPUMCTX_EXTRN_HM_VMX_MASK)
109
110	/**
111	* Exception bitmap mask for real-mode guests (real-on-v86).
112	*
113	* We need to intercept all exceptions manually except:
114	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
115	* due to bugs in Intel CPUs.
116	* - \#PF need not be intercepted even in real-mode if we have nested paging
117	* support.
118	*/
119	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
120	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
121	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
122	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
123	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
124	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
125	\| RT_BIT(X86_XCPT_XF))
126
127	/** Maximum VM-instruction error number. */
128	#define HMVMX_INSTR_ERROR_MAX 28
129
130	/** Profiling macro. */
131	#ifdef HM_PROFILE_EXIT_DISPATCH
132	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
133	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
134	#else
135	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
136	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
137	#endif
138
139	/** Assert that preemption is disabled or covered by thread-context hooks. */
140	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
141	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
142
143	/** Assert that we haven't migrated CPUs when thread-context hooks are not
144	* used. */
145	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
146	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
147	("Illegal migration! Entered on CPU %u Current %u\n", \
148	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
149
150	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
151	* context. */
152	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
153	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
154	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
155
156	/** Log the VM-exit reason with an easily visible marker to identify it in a
157	* potential sea of logging data. */
158	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
159	do { \
160	Log4(("VM-exit: vcpu[%RU32] reason=%#x -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", \
161	(a_pVCpu)->idCpu, (a_uExitReason))); \
162	} while (0) \
163
164
165	/*********************************************************************************************************************************
166	* Structures and Typedefs *
167	*********************************************************************************************************************************/
168	/**
169	* VMX per-VCPU transient state.
170	*
171	* A state structure for holding miscellaneous information across
172	* VMX non-root operation and restored after the transition.
173	*/
174	typedef struct VMXTRANSIENT
175	{
176	/** The host's rflags/eflags. */
177	RTCCUINTREG fEFlags;
178	#if HC_ARCH_BITS == 32
179	uint32_t u32Alignment0;
180	#endif
181	/** The guest's TPR value used for TPR shadowing. */
182	uint8_t u8GuestTpr;
183	/** Alignment. */
184	uint8_t abAlignment0[7];
185
186	/** The basic VM-exit reason. */
187	uint16_t uExitReason;
188	/** Alignment. */
189	uint16_t u16Alignment0;
190	/** The VM-exit interruption error code. */
191	uint32_t uExitIntErrorCode;
192	/** The VM-exit exit code qualification. */
193	uint64_t uExitQual;
194	/** The Guest-linear address. */
195	uint64_t uGuestLinearAddr;
196
197	/** The VM-exit interruption-information field. */
198	uint32_t uExitIntInfo;
199	/** The VM-exit instruction-length field. */
200	uint32_t cbInstr;
201	/** The VM-exit instruction-information field. */
202	VMXEXITINSTRINFO ExitInstrInfo;
203	/** Whether the VM-entry failed or not. */
204	bool fVMEntryFailed;
205	/** Whether we are currently executing a nested-guest. */
206	bool fIsNestedGuest;
207	/** Alignment. */
208	uint8_t abAlignment1[2];
209
210	/** The VM-entry interruption-information field. */
211	uint32_t uEntryIntInfo;
212	/** The VM-entry exception error code field. */
213	uint32_t uEntryXcptErrorCode;
214	/** The VM-entry instruction length field. */
215	uint32_t cbEntryInstr;
216
217	/** IDT-vectoring information field. */
218	uint32_t uIdtVectoringInfo;
219	/** IDT-vectoring error code. */
220	uint32_t uIdtVectoringErrorCode;
221
222	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
223	uint32_t fVmcsFieldsRead;
224
225	/** Whether the guest debug state was active at the time of VM-exit. */
226	bool fWasGuestDebugStateActive;
227	/** Whether the hyper debug state was active at the time of VM-exit. */
228	bool fWasHyperDebugStateActive;
229	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
230	bool fUpdatedTscOffsettingAndPreemptTimer;
231	/** Whether the VM-exit was caused by a page-fault during delivery of a
232	* contributory exception or a page-fault. */
233	bool fVectoringDoublePF;
234	/** Whether the VM-exit was caused by a page-fault during delivery of an
235	* external interrupt or NMI. */
236	bool fVectoringPF;
237	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
238	* area after VM-exit. */
239	bool fRemoveTscAuxMsr;
240	bool afAlignment0[2];
241
242	/** The VMCS info. object. */
243	PVMXVMCSINFO pVmcsInfo;
244	} VMXTRANSIENT;
245	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
246	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
247	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
248	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
249	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
250	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
251	/** Pointer to VMX transient state. */
252	typedef VMXTRANSIENT *PVMXTRANSIENT;
253	/** Pointer to a const VMX transient state. */
254	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
255
256	/**
257	* Memory operand read or write access.
258	*/
259	typedef enum VMXMEMACCESS
260	{
261	VMXMEMACCESS_READ = 0,
262	VMXMEMACCESS_WRITE = 1
263	} VMXMEMACCESS;
264
265	/**
266	* VMX VM-exit handler.
267	*
268	* @returns Strict VBox status code (i.e. informational status codes too).
269	* @param pVCpu The cross context virtual CPU structure.
270	* @param pVmxTransient The VMX-transient structure.
271	*/
272	#ifndef HMVMX_USE_FUNCTION_TABLE
273	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
274	#else
275	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
276	/** Pointer to VM-exit handler. */
277	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
278	#endif
279
280	/**
281	* VMX VM-exit handler, non-strict status code.
282	*
283	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
284	*
285	* @returns VBox status code, no informational status code returned.
286	* @param pVCpu The cross context virtual CPU structure.
287	* @param pVmxTransient The VMX-transient structure.
288	*
289	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
290	* use of that status code will be replaced with VINF_EM_SOMETHING
291	* later when switching over to IEM.
292	*/
293	#ifndef HMVMX_USE_FUNCTION_TABLE
294	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
295	#else
296	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
297	#endif
298
299
300	/*********************************************************************************************************************************
301	* Internal Functions *
302	*********************************************************************************************************************************/
303	#ifndef HMVMX_USE_FUNCTION_TABLE
304	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
305	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
306	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
307	#else
308	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
309	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
310	#endif
311	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
312	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
313	#endif
314
315	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
316	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
317	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
318	#endif
319
320	/** @name VM-exit handler prototypes.
321	* @{
322	*/
323	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
324	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
325	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
326	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
327	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
328	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
329	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
330	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
331	static FNVMXEXITHANDLER hmR0VmxExitHlt;
332	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
333	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
334	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
335	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
336	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
337	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
338	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
339	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
340	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
341	static FNVMXEXITHANDLER hmR0VmxExitVmread;
342	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
343	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
344	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
345	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
346	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
347	#endif
348	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
349	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
350	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
351	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
352	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
353	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
354	static FNVMXEXITHANDLER hmR0VmxExitMwait;
355	static FNVMXEXITHANDLER hmR0VmxExitMtf;
356	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
357	static FNVMXEXITHANDLER hmR0VmxExitPause;
358	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
359	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
360	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
361	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
362	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
363	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
365	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
366	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
370	/** @} */
371
372	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
373	/** @name Nested-guest VM-exit handler prototypes.
374	* @{
375	*/
376	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
377	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
378	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
379	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
380	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
381	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
382	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
383	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
384	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
385	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
386	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
387	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
388	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
389	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
390	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
391	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
392	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
393	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
394	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
395	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
396	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
397	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
398	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
399	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
401	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
404	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
405	/** @} */
406	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
407
408
409	/*********************************************************************************************************************************
410	* Global Variables *
411	*********************************************************************************************************************************/
412	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
413	/**
414	* Array of all VMCS fields.
415	* Any fields added to the VT-x spec. should be added here.
416	*
417	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
418	* of nested-guests.
419	*/
420	static const uint32_t g_aVmcsFields[] =
421	{
422	/* 16-bit control fields. */
423	VMX_VMCS16_VPID,
424	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
425	VMX_VMCS16_EPTP_INDEX,
426
427	/* 16-bit guest-state fields. */
428	VMX_VMCS16_GUEST_ES_SEL,
429	VMX_VMCS16_GUEST_CS_SEL,
430	VMX_VMCS16_GUEST_SS_SEL,
431	VMX_VMCS16_GUEST_DS_SEL,
432	VMX_VMCS16_GUEST_FS_SEL,
433	VMX_VMCS16_GUEST_GS_SEL,
434	VMX_VMCS16_GUEST_LDTR_SEL,
435	VMX_VMCS16_GUEST_TR_SEL,
436	VMX_VMCS16_GUEST_INTR_STATUS,
437	VMX_VMCS16_GUEST_PML_INDEX,
438
439	/* 16-bits host-state fields. */
440	VMX_VMCS16_HOST_ES_SEL,
441	VMX_VMCS16_HOST_CS_SEL,
442	VMX_VMCS16_HOST_SS_SEL,
443	VMX_VMCS16_HOST_DS_SEL,
444	VMX_VMCS16_HOST_FS_SEL,
445	VMX_VMCS16_HOST_GS_SEL,
446	VMX_VMCS16_HOST_TR_SEL,
447
448	/* 64-bit control fields. */
449	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
450	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
451	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
452	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
453	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
454	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
455	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
456	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
457	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
458	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
459	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
460	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
461	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
462	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
463	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
464	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
465	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
466	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
467	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
468	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
469	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
470	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
471	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
472	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
473	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
474	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
475	VMX_VMCS64_CTRL_EPTP_FULL,
476	VMX_VMCS64_CTRL_EPTP_HIGH,
477	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
478	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
479	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
480	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
481	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
482	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
483	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
484	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
485	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
486	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
487	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
488	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
489	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
490	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
491	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
492	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
493	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
494	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
495	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
496	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
497	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
498	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
499
500	/* 64-bit read-only data fields. */
501	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
502	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
503
504	/* 64-bit guest-state fields. */
505	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
506	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
507	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
508	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
509	VMX_VMCS64_GUEST_PAT_FULL,
510	VMX_VMCS64_GUEST_PAT_HIGH,
511	VMX_VMCS64_GUEST_EFER_FULL,
512	VMX_VMCS64_GUEST_EFER_HIGH,
513	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
514	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
515	VMX_VMCS64_GUEST_PDPTE0_FULL,
516	VMX_VMCS64_GUEST_PDPTE0_HIGH,
517	VMX_VMCS64_GUEST_PDPTE1_FULL,
518	VMX_VMCS64_GUEST_PDPTE1_HIGH,
519	VMX_VMCS64_GUEST_PDPTE2_FULL,
520	VMX_VMCS64_GUEST_PDPTE2_HIGH,
521	VMX_VMCS64_GUEST_PDPTE3_FULL,
522	VMX_VMCS64_GUEST_PDPTE3_HIGH,
523	VMX_VMCS64_GUEST_BNDCFGS_FULL,
524	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
525
526	/* 64-bit host-state fields. */
527	VMX_VMCS64_HOST_PAT_FULL,
528	VMX_VMCS64_HOST_PAT_HIGH,
529	VMX_VMCS64_HOST_EFER_FULL,
530	VMX_VMCS64_HOST_EFER_HIGH,
531	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
532	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
533
534	/* 32-bit control fields. */
535	VMX_VMCS32_CTRL_PIN_EXEC,
536	VMX_VMCS32_CTRL_PROC_EXEC,
537	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
538	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
539	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
540	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
541	VMX_VMCS32_CTRL_EXIT,
542	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
543	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
544	VMX_VMCS32_CTRL_ENTRY,
545	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
546	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
547	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
548	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
549	VMX_VMCS32_CTRL_TPR_THRESHOLD,
550	VMX_VMCS32_CTRL_PROC_EXEC2,
551	VMX_VMCS32_CTRL_PLE_GAP,
552	VMX_VMCS32_CTRL_PLE_WINDOW,
553
554	/* 32-bits read-only fields. */
555	VMX_VMCS32_RO_VM_INSTR_ERROR,
556	VMX_VMCS32_RO_EXIT_REASON,
557	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
558	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
559	VMX_VMCS32_RO_IDT_VECTORING_INFO,
560	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
561	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
562	VMX_VMCS32_RO_EXIT_INSTR_INFO,
563
564	/* 32-bit guest-state fields. */
565	VMX_VMCS32_GUEST_ES_LIMIT,
566	VMX_VMCS32_GUEST_CS_LIMIT,
567	VMX_VMCS32_GUEST_SS_LIMIT,
568	VMX_VMCS32_GUEST_DS_LIMIT,
569	VMX_VMCS32_GUEST_FS_LIMIT,
570	VMX_VMCS32_GUEST_GS_LIMIT,
571	VMX_VMCS32_GUEST_LDTR_LIMIT,
572	VMX_VMCS32_GUEST_TR_LIMIT,
573	VMX_VMCS32_GUEST_GDTR_LIMIT,
574	VMX_VMCS32_GUEST_IDTR_LIMIT,
575	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
576	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
577	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
578	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
579	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
580	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
581	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
582	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
583	VMX_VMCS32_GUEST_INT_STATE,
584	VMX_VMCS32_GUEST_ACTIVITY_STATE,
585	VMX_VMCS32_GUEST_SMBASE,
586	VMX_VMCS32_GUEST_SYSENTER_CS,
587	VMX_VMCS32_PREEMPT_TIMER_VALUE,
588
589	/* 32-bit host-state fields. */
590	VMX_VMCS32_HOST_SYSENTER_CS,
591
592	/* Natural-width control fields. */
593	VMX_VMCS_CTRL_CR0_MASK,
594	VMX_VMCS_CTRL_CR4_MASK,
595	VMX_VMCS_CTRL_CR0_READ_SHADOW,
596	VMX_VMCS_CTRL_CR4_READ_SHADOW,
597	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
598	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
599	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
600	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
601
602	/* Natural-width read-only data fields. */
603	VMX_VMCS_RO_EXIT_QUALIFICATION,
604	VMX_VMCS_RO_IO_RCX,
605	VMX_VMCS_RO_IO_RSI,
606	VMX_VMCS_RO_IO_RDI,
607	VMX_VMCS_RO_IO_RIP,
608	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
609
610	/* Natural-width guest-state field */
611	VMX_VMCS_GUEST_CR0,
612	VMX_VMCS_GUEST_CR3,
613	VMX_VMCS_GUEST_CR4,
614	VMX_VMCS_GUEST_ES_BASE,
615	VMX_VMCS_GUEST_CS_BASE,
616	VMX_VMCS_GUEST_SS_BASE,
617	VMX_VMCS_GUEST_DS_BASE,
618	VMX_VMCS_GUEST_FS_BASE,
619	VMX_VMCS_GUEST_GS_BASE,
620	VMX_VMCS_GUEST_LDTR_BASE,
621	VMX_VMCS_GUEST_TR_BASE,
622	VMX_VMCS_GUEST_GDTR_BASE,
623	VMX_VMCS_GUEST_IDTR_BASE,
624	VMX_VMCS_GUEST_DR7,
625	VMX_VMCS_GUEST_RSP,
626	VMX_VMCS_GUEST_RIP,
627	VMX_VMCS_GUEST_RFLAGS,
628	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
629	VMX_VMCS_GUEST_SYSENTER_ESP,
630	VMX_VMCS_GUEST_SYSENTER_EIP,
631
632	/* Natural-width host-state fields */
633	VMX_VMCS_HOST_CR0,
634	VMX_VMCS_HOST_CR3,
635	VMX_VMCS_HOST_CR4,
636	VMX_VMCS_HOST_FS_BASE,
637	VMX_VMCS_HOST_GS_BASE,
638	VMX_VMCS_HOST_TR_BASE,
639	VMX_VMCS_HOST_GDTR_BASE,
640	VMX_VMCS_HOST_IDTR_BASE,
641	VMX_VMCS_HOST_SYSENTER_ESP,
642	VMX_VMCS_HOST_SYSENTER_EIP,
643	VMX_VMCS_HOST_RSP,
644	VMX_VMCS_HOST_RIP
645	};
646	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
647
648	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
649	static const uint32_t g_aVmcsCacheSegBase[] =
650	{
651	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
652	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
653	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
654	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
655	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
656	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
657	};
658	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
659	#endif
660	static const uint32_t g_aVmcsSegBase[] =
661	{
662	VMX_VMCS_GUEST_ES_BASE,
663	VMX_VMCS_GUEST_CS_BASE,
664	VMX_VMCS_GUEST_SS_BASE,
665	VMX_VMCS_GUEST_DS_BASE,
666	VMX_VMCS_GUEST_FS_BASE,
667	VMX_VMCS_GUEST_GS_BASE
668	};
669	static const uint32_t g_aVmcsSegSel[] =
670	{
671	VMX_VMCS16_GUEST_ES_SEL,
672	VMX_VMCS16_GUEST_CS_SEL,
673	VMX_VMCS16_GUEST_SS_SEL,
674	VMX_VMCS16_GUEST_DS_SEL,
675	VMX_VMCS16_GUEST_FS_SEL,
676	VMX_VMCS16_GUEST_GS_SEL
677	};
678	static const uint32_t g_aVmcsSegLimit[] =
679	{
680	VMX_VMCS32_GUEST_ES_LIMIT,
681	VMX_VMCS32_GUEST_CS_LIMIT,
682	VMX_VMCS32_GUEST_SS_LIMIT,
683	VMX_VMCS32_GUEST_DS_LIMIT,
684	VMX_VMCS32_GUEST_FS_LIMIT,
685	VMX_VMCS32_GUEST_GS_LIMIT
686	};
687	static const uint32_t g_aVmcsSegAttr[] =
688	{
689	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
690	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
691	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
692	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
693	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
694	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
695	};
696	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
697	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
698	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
699	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
700
701	#ifdef HMVMX_USE_FUNCTION_TABLE
702	/**
703	* VMX_EXIT dispatch table.
704	*/
705	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
706	{
707	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
708	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
709	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
710	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
711	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
712	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
713	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
714	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
715	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
716	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
717	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
718	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
719	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
720	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
721	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
722	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
723	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
724	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
725	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
726	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
727	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
728	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
729	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
730	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
731	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
732	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
733	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
734	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
735	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
736	#else
737	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
738	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
739	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
740	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
741	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
742	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
743	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
744	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
745	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
746	#endif
747	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
748	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
749	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
750	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
751	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
752	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
753	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
754	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
755	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
756	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
757	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
758	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
759	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
760	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
761	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
762	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
763	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
764	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
765	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
766	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
767	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
768	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
769	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
770	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
771	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
772	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
773	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
774	#else
775	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
776	#endif
777	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
778	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
779	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
780	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
781	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
782	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
783	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
784	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
785	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
786	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
787	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
788	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
789	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
790	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
791	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
792	};
793	#endif /* HMVMX_USE_FUNCTION_TABLE */
794
795	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
796	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
797	{
798	/* 0 */ "(Not Used)",
799	/* 1 */ "VMCALL executed in VMX root operation.",
800	/* 2 */ "VMCLEAR with invalid physical address.",
801	/* 3 */ "VMCLEAR with VMXON pointer.",
802	/* 4 */ "VMLAUNCH with non-clear VMCS.",
803	/* 5 */ "VMRESUME with non-launched VMCS.",
804	/* 6 */ "VMRESUME after VMXOFF",
805	/* 7 */ "VM-entry with invalid control fields.",
806	/* 8 */ "VM-entry with invalid host state fields.",
807	/* 9 */ "VMPTRLD with invalid physical address.",
808	/* 10 */ "VMPTRLD with VMXON pointer.",
809	/* 11 */ "VMPTRLD with incorrect revision identifier.",
810	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
811	/* 13 */ "VMWRITE to read-only VMCS component.",
812	/* 14 */ "(Not Used)",
813	/* 15 */ "VMXON executed in VMX root operation.",
814	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
815	/* 17 */ "VM-entry with non-launched executing VMCS.",
816	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
817	/* 19 */ "VMCALL with non-clear VMCS.",
818	/* 20 */ "VMCALL with invalid VM-exit control fields.",
819	/* 21 */ "(Not Used)",
820	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
821	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
822	/* 24 */ "VMCALL with invalid SMM-monitor features.",
823	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
824	/* 26 */ "VM-entry with events blocked by MOV SS.",
825	/* 27 */ "(Not Used)",
826	/* 28 */ "Invalid operand to INVEPT/INVVPID."
827	};
828	#endif /* VBOX_STRICT && LOG_ENABLED */
829
830
831	/**
832	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
833	*
834	* Any bit set in this mask is owned by the host/hypervisor and would cause a
835	* VM-exit when modified by the guest.
836	*
837	* @returns The static CR0 guest/host mask.
838	* @param pVCpu The cross context virtual CPU structure.
839	*/
840	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
841	{
842	/*
843	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
844	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
845	*/
846	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
847	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
848	* and @bugref{6944}. */
849	PVM pVM = pVCpu->CTX_SUFF(pVM);
850	return ( X86_CR0_PE
851	\| X86_CR0_NE
852	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
853	\| X86_CR0_PG
854	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
855	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
856	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
857	}
858
859
860	/**
861	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
862	*
863	* Any bit set in this mask is owned by the host/hypervisor and would cause a
864	* VM-exit when modified by the guest.
865	*
866	* @returns The static CR4 guest/host mask.
867	* @param pVCpu The cross context virtual CPU structure.
868	*/
869	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
870	{
871	/*
872	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
873	* these bits are reserved on hardware that does not support them. Since the
874	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
875	* these bits and handle it depending on whether we expose them to the guest.
876	*/
877	PVM pVM = pVCpu->CTX_SUFF(pVM);
878	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
879	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
880	return ( X86_CR4_VMXE
881	\| X86_CR4_VME
882	\| X86_CR4_PAE
883	\| X86_CR4_PGE
884	\| X86_CR4_PSE
885	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
886	\| (fPcid ? X86_CR4_PCIDE : 0));
887	}
888
889
890	/**
891	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
892	* area.
893	*
894	* @returns @c true if it's different, @c false otherwise.
895	* @param pVmcsInfo The VMCS info. object.
896	*/
897	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
898	{
899	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
900	&& pVmcsInfo->pvGuestMsrStore);
901	}
902
903
904	/**
905	* Checks whether one of the given Pin-based VM-execution controls are set.
906	*
907	* @returns @c true if set, @c false otherwise.
908	* @param pVCpu The cross context virtual CPU structure.
909	* @param pVmxTransient The VMX-transient structure.
910	* @param uPinCtls The Pin-based VM-execution controls to check.
911	*
912	* @remarks This will not check merged controls when executing a nested-guest
913	* but the original control specified by the guest hypervisor.
914	*/
915	static bool hmR0VmxIsPinCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uPinCtls)
916	{
917	if (!pVmxTransient->fIsNestedGuest)
918	{
919	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
920	return RT_BOOL(pVmcsInfo->u32PinCtls & uPinCtls);
921	}
922	return CPUMIsGuestVmxPinCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uPinCtls);
923	}
924
925
926	#if 0
927	/**
928	* Checks whether one of the given Processor-based VM-execution controls are set.
929	*
930	* @returns @c true if set, @c false otherwise.
931	* @param pVCpu The cross context virtual CPU structure.
932	* @param pVmxTransient The VMX-transient structure.
933	* @param uProcCtls The Processor-based VM-execution controls to check.
934	*
935	* @remarks This will not check merged controls when executing a nested-guest
936	* but the original control specified by the guest hypervisor.
937	*/
938	static bool hmR0VmxIsProcCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
939	{
940	if (!pVmxTransient->fIsNestedGuest)
941	{
942	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
943	return RT_BOOL(pVmcsInfo->u32ProcCtls & uProcCtls);
944	}
945	return CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
946	}
947
948
949	/**
950	* Checks whether one of the given Secondary Processor-based VM-execution controls
951	* are set.
952	*
953	* @returns @c true if set, @c false otherwise.
954	* @param pVCpu The cross context virtual CPU structure.
955	* @param pVmxTransient The VMX-transient structure.
956	* @param uProcCtls2 The Secondary Processor-based VM-execution controls to
957	* check.
958	*
959	* @remarks This will not check merged controls when executing a nested-guest
960	* but the original control specified by the guest hypervisor.
961	*/
962	static bool hmR0VmxIsProcCtls2Set(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uProcCtls2)
963	{
964	if (!pVmxTransient->fIsNestedGuest)
965	{
966	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
967	return RT_BOOL(pVmcsInfo->u32ProcCtls2 & uProcCtls2);
968	}
969	return CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls2);
970	}
971
972
973	/**
974	* Checks whether one of the given VM-entry controls are set.
975	*
976	* @returns @c true if set, @c false otherwise.
977	* @param pVCpu The cross context virtual CPU structure.
978	* @param pVmxTransient The VMX-transient structure.
979	* @param uEntryCtls The VM-entry controls to check.
980	*
981	* @remarks This will not check merged controls when executing a nested-guest
982	* but the original control specified by the guest hypervisor.
983	*/
984	static bool hmR0VmxIsEntryCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uEntryCtls)
985	{
986	if (!pVmxTransient->fIsNestedGuest)
987	{
988	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
989	return RT_BOOL(pVmcsInfo->u32EntryCtls & uEntryCtls);
990	}
991	return CPUMIsGuestVmxEntryCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uEntryCtls);
992	}
993
994
995	/**
996	* Checks whether one of the given VM-exit controls are set.
997	*
998	* @returns @c true if set, @c false otherwise.
999	* @param pVCpu The cross context virtual CPU structure.
1000	* @param pVmxTransient The VMX-transient structure.
1001	* @param uExitCtls The VM-exit controls to check.
1002	*
1003	* @remarks This will not check merged controls when executing a nested-guest
1004	* but the original control specified by the guest hypervisor.
1005	*/
1006	static bool hmR0VmxIsExitCtlsSet(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uExitCtls)
1007	{
1008	if (!pVmxTransient->fIsNestedGuest)
1009	{
1010	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1011	return RT_BOOL(pVmcsInfo->u32ExitCtls & uExitCtls);
1012	}
1013	return CPUMIsGuestVmxExitCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uExitCtls);
1014	}
1015	#endif
1016
1017
1018	/**
1019	* Adds one or more exceptions to the exception bitmap and commits it to the current
1020	* VMCS.
1021	*
1022	* @returns VBox status code.
1023	* @param pVmxTransient The VMX-transient structure.
1024	* @param uXcptMask The exception(s) to add.
1025	*/
1026	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1027	{
1028	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1029	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
1030	if ((uXcptBitmap & uXcptMask) != uXcptMask)
1031	{
1032	uXcptBitmap \|= uXcptMask;
1033	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
1034	AssertRCReturn(rc, rc);
1035	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1036	}
1037	return VINF_SUCCESS;
1038	}
1039
1040
1041	/**
1042	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1043	*
1044	* @returns VBox status code.
1045	* @param pVmxTransient The VMX-transient structure.
1046	* @param uXcpt The exception to add.
1047	*/
1048	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1049	{
1050	Assert(uXcpt <= X86_XCPT_LAST);
1051	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1052	}
1053
1054
1055	/**
1056	* Remove one or more exceptions from the exception bitmap and commits it to the
1057	* current VMCS.
1058	*
1059	* This takes care of not removing the exception intercept if a nested-guest
1060	* requires the exception to be intercepted.
1061	*
1062	* @returns VBox status code.
1063	* @param pVCpu The cross context virtual CPU structure.
1064	* @param pVmxTransient The VMX-transient structure.
1065	* @param uXcptMask The exception(s) to remove.
1066	*/
1067	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1068	{
1069	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1070	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1071	if (u32XcptBitmap & uXcptMask)
1072	{
1073	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1074	if (!pVmxTransient->fIsNestedGuest)
1075	{ /* likely */ }
1076	else
1077	{
1078	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1079	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1080	}
1081	#endif
1082	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1083	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1084	\| RT_BIT(X86_XCPT_DE)
1085	\| RT_BIT(X86_XCPT_NM)
1086	\| RT_BIT(X86_XCPT_TS)
1087	\| RT_BIT(X86_XCPT_UD)
1088	\| RT_BIT(X86_XCPT_NP)
1089	\| RT_BIT(X86_XCPT_SS)
1090	\| RT_BIT(X86_XCPT_GP)
1091	\| RT_BIT(X86_XCPT_PF)
1092	\| RT_BIT(X86_XCPT_MF));
1093	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1094	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1095	#endif
1096	if (uXcptMask)
1097	{
1098	/* Validate we are not removing any essential exception intercepts. */
1099	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1100	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1101	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1102
1103	/* Remove it from the exception bitmap. */
1104	u32XcptBitmap &= ~uXcptMask;
1105
1106	/* Commit and update the cache if necessary. */
1107	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1108	{
1109	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1110	AssertRCReturn(rc, rc);
1111	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1112	}
1113	}
1114	}
1115	return VINF_SUCCESS;
1116	}
1117
1118
1119	/**
1120	* Remove an exceptions from the exception bitmap and commits it to the current
1121	* VMCS.
1122	*
1123	* @returns VBox status code.
1124	* @param pVCpu The cross context virtual CPU structure.
1125	* @param pVmxTransient The VMX-transient structure.
1126	* @param uXcpt The exception to remove.
1127	*/
1128	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1129	{
1130	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1131	}
1132
1133
1134	/**
1135	* Loads the VMCS specified by the VMCS info. object.
1136	*
1137	* @returns VBox status code.
1138	* @param pVmcsInfo The VMCS info. object.
1139	*
1140	* @remarks Can be called with interrupts disabled.
1141	*/
1142	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1143	{
1144	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1145	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1146
1147	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1148	if (RT_SUCCESS(rc))
1149	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1150	return rc;
1151	}
1152
1153
1154	/**
1155	* Clears the VMCS specified by the VMCS info. object.
1156	*
1157	* @returns VBox status code.
1158	* @param pVmcsInfo The VMCS info. object.
1159	*
1160	* @remarks Can be called with interrupts disabled.
1161	*/
1162	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1163	{
1164	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1165	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1166
1167	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1168	if (RT_SUCCESS(rc))
1169	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1170	return rc;
1171	}
1172
1173
1174	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1175	/**
1176	* Loads the shadow VMCS specified by the VMCS info. object.
1177	*
1178	* @returns VBox status code.
1179	* @param pVmcsInfo The VMCS info. object.
1180	*
1181	* @remarks Can be called with interrupts disabled.
1182	*/
1183	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1184	{
1185	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1186	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1187
1188	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1189	if (RT_SUCCESS(rc))
1190	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1191	return rc;
1192	}
1193
1194
1195	/**
1196	* Clears the shadow VMCS specified by the VMCS info. object.
1197	*
1198	* @returns VBox status code.
1199	* @param pVmcsInfo The VMCS info. object.
1200	*
1201	* @remarks Can be called with interrupts disabled.
1202	*/
1203	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1204	{
1205	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1206	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1207
1208	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1209	if (RT_SUCCESS(rc))
1210	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1211	return rc;
1212	}
1213
1214
1215	/**
1216	* Switches from and to the specified VMCSes.
1217	*
1218	* @returns VBox status code.
1219	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1220	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1221	*
1222	* @remarks Called with interrupts disabled.
1223	*/
1224	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1225	{
1226	/*
1227	* Clear the VMCS we are switching out if it has not already been cleared.
1228	* This will sync any CPU internal data back to the VMCS.
1229	*/
1230	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1231	{
1232	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1233	if (RT_SUCCESS(rc))
1234	{
1235	/*
1236	* The shadow VMCS, if any, would not be active at this point since we
1237	* would have cleared it while importing the virtual hardware-virtualization
1238	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1239	* clear the shadow VMCS here, just assert for safety.
1240	*/
1241	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1242	}
1243	else
1244	return rc;
1245	}
1246
1247	/*
1248	* Clear the VMCS we are switching to if it has not already been cleared.
1249	* This will initialize the VMCS launch state to "clear" required for loading it.
1250	*
1251	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1252	*/
1253	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1254	{
1255	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1256	if (RT_SUCCESS(rc))
1257	{ /* likely */ }
1258	else
1259	return rc;
1260	}
1261
1262	/*
1263	* Finally, load the VMCS we are switching to.
1264	*/
1265	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1266	}
1267
1268
1269	/**
1270	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1271	* caller.
1272	*
1273	* @returns VBox status code.
1274	* @param pVCpu The cross context virtual CPU structure.
1275	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1276	* true) or guest VMCS (pass false).
1277	*/
1278	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1279	{
1280	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1281	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1282
1283	PVMXVMCSINFO pVmcsInfoFrom;
1284	PVMXVMCSINFO pVmcsInfoTo;
1285	if (fSwitchToNstGstVmcs)
1286	{
1287	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1288	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1289	}
1290	else
1291	{
1292	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1293	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1294	}
1295
1296	/*
1297	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1298	* preemption hook code path acquires the current VMCS.
1299	*/
1300	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1301
1302	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1303	if (RT_SUCCESS(rc))
1304	{
1305	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1306
1307	/*
1308	* If we are switching to a VMCS that was executed on a different host CPU or was
1309	* never executed before, flag that we need to export the host state before executing
1310	* guest/nested-guest code using hardware-assisted VMX.
1311	*
1312	* This could probably be done in a preemptible context since the preemption hook
1313	* will flag the necessary change in host context. However, since preemption is
1314	* already disabled and to avoid making assumptions about host specific code in
1315	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1316	* disabled.
1317	*/
1318	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1319	{ /* likely */ }
1320	else
1321	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1322
1323	ASMSetFlags(fEFlags);
1324
1325	/*
1326	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1327	* flag that we need to update the host MSR values there. Even if we decide in the
1328	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1329	* if its content differs, we would have to update the host MSRs anyway.
1330	*/
1331	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1332	}
1333	else
1334	ASMSetFlags(fEFlags);
1335	return rc;
1336	}
1337	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1338
1339
1340	/**
1341	* Updates the VM's last error record.
1342	*
1343	* If there was a VMX instruction error, reads the error data from the VMCS and
1344	* updates VCPU's last error record as well.
1345	*
1346	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1347	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1348	* VERR_VMX_INVALID_VMCS_FIELD.
1349	* @param rc The error code.
1350	*/
1351	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1352	{
1353	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1354	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1355	{
1356	AssertPtrReturnVoid(pVCpu);
1357	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1358	}
1359	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1360	}
1361
1362
1363	#ifdef VBOX_STRICT
1364	/**
1365	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1366	* transient structure.
1367	*
1368	* @returns VBox status code.
1369	* @param pVmxTransient The VMX-transient structure.
1370	*
1371	* @remarks No-long-jump zone!!!
1372	*/
1373	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1374	{
1375	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1376	AssertRCReturn(rc, rc);
1377	return VINF_SUCCESS;
1378	}
1379
1380
1381	/**
1382	* Reads the VM-entry exception error code field from the VMCS into
1383	* the VMX transient structure.
1384	*
1385	* @returns VBox status code.
1386	* @param pVmxTransient The VMX-transient structure.
1387	*
1388	* @remarks No-long-jump zone!!!
1389	*/
1390	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1391	{
1392	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1393	AssertRCReturn(rc, rc);
1394	return VINF_SUCCESS;
1395	}
1396
1397
1398	/**
1399	* Reads the VM-entry exception error code field from the VMCS into
1400	* the VMX transient structure.
1401	*
1402	* @returns VBox status code.
1403	* @param pVmxTransient The VMX-transient structure.
1404	*
1405	* @remarks No-long-jump zone!!!
1406	*/
1407	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1408	{
1409	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1410	AssertRCReturn(rc, rc);
1411	return VINF_SUCCESS;
1412	}
1413	#endif /* VBOX_STRICT */
1414
1415
1416	/**
1417	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1418	* transient structure.
1419	*
1420	* @returns VBox status code.
1421	* @param pVmxTransient The VMX-transient structure.
1422	*/
1423	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1424	{
1425	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1426	{
1427	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1428	AssertRCReturn(rc,rc);
1429	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1430	}
1431	return VINF_SUCCESS;
1432	}
1433
1434
1435	/**
1436	* Reads the VM-exit interruption error code from the VMCS into the VMX
1437	* transient structure.
1438	*
1439	* @returns VBox status code.
1440	* @param pVmxTransient The VMX-transient structure.
1441	*/
1442	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1443	{
1444	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1445	{
1446	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1447	AssertRCReturn(rc, rc);
1448	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1449	}
1450	return VINF_SUCCESS;
1451	}
1452
1453
1454	/**
1455	* Reads the VM-exit instruction length field from the VMCS into the VMX
1456	* transient structure.
1457	*
1458	* @returns VBox status code.
1459	* @param pVmxTransient The VMX-transient structure.
1460	*/
1461	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1462	{
1463	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1464	{
1465	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1466	AssertRCReturn(rc, rc);
1467	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1468	}
1469	return VINF_SUCCESS;
1470	}
1471
1472
1473	/**
1474	* Reads the VM-exit instruction-information field from the VMCS into
1475	* the VMX transient structure.
1476	*
1477	* @returns VBox status code.
1478	* @param pVmxTransient The VMX-transient structure.
1479	*/
1480	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1481	{
1482	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1483	{
1484	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1485	AssertRCReturn(rc, rc);
1486	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1487	}
1488	return VINF_SUCCESS;
1489	}
1490
1491
1492	/**
1493	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1494	*
1495	* @returns VBox status code.
1496	* @param pVCpu The cross context virtual CPU structure of the
1497	* calling EMT. (Required for the VMCS cache case.)
1498	* @param pVmxTransient The VMX-transient structure.
1499	*/
1500	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1501	{
1502	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1503	{
1504	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1505	AssertRCReturn(rc, rc);
1506	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1507	}
1508	return VINF_SUCCESS;
1509	}
1510
1511
1512	/**
1513	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1514	*
1515	* @returns VBox status code.
1516	* @param pVCpu The cross context virtual CPU structure of the
1517	* calling EMT. (Required for the VMCS cache case.)
1518	* @param pVmxTransient The VMX-transient structure.
1519	*/
1520	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1521	{
1522	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1523	{
1524	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1525	AssertRCReturn(rc, rc);
1526	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1527	}
1528	return VINF_SUCCESS;
1529	}
1530
1531
1532	/**
1533	* Reads the IDT-vectoring information field from the VMCS into the VMX
1534	* transient structure.
1535	*
1536	* @returns VBox status code.
1537	* @param pVmxTransient The VMX-transient structure.
1538	*
1539	* @remarks No-long-jump zone!!!
1540	*/
1541	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1542	{
1543	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1544	{
1545	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1546	AssertRCReturn(rc, rc);
1547	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1548	}
1549	return VINF_SUCCESS;
1550	}
1551
1552
1553	/**
1554	* Reads the IDT-vectoring error code from the VMCS into the VMX
1555	* transient structure.
1556	*
1557	* @returns VBox status code.
1558	* @param pVmxTransient The VMX-transient structure.
1559	*/
1560	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1561	{
1562	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1563	{
1564	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1565	AssertRCReturn(rc, rc);
1566	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1567	}
1568	return VINF_SUCCESS;
1569	}
1570
1571
1572	/**
1573	* Enters VMX root mode operation on the current CPU.
1574	*
1575	* @returns VBox status code.
1576	* @param pVM The cross context VM structure. Can be
1577	* NULL, after a resume.
1578	* @param HCPhysCpuPage Physical address of the VMXON region.
1579	* @param pvCpuPage Pointer to the VMXON region.
1580	*/
1581	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1582	{
1583	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1584	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1585	Assert(pvCpuPage);
1586	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1587
1588	if (pVM)
1589	{
1590	/* Write the VMCS revision identifier to the VMXON region. */
1591	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1592	}
1593
1594	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1595	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1596
1597	/* Enable the VMX bit in CR4 if necessary. */
1598	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1599
1600	/* Enter VMX root mode. */
1601	int rc = VMXEnable(HCPhysCpuPage);
1602	if (RT_FAILURE(rc))
1603	{
1604	if (!(uOldCr4 & X86_CR4_VMXE))
1605	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1606
1607	if (pVM)
1608	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1609	}
1610
1611	/* Restore interrupts. */
1612	ASMSetFlags(fEFlags);
1613	return rc;
1614	}
1615
1616
1617	/**
1618	* Exits VMX root mode operation on the current CPU.
1619	*
1620	* @returns VBox status code.
1621	*/
1622	static int hmR0VmxLeaveRootMode(void)
1623	{
1624	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1625
1626	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1627	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1628
1629	/* If we're for some reason not in VMX root mode, then don't leave it. */
1630	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1631
1632	int rc;
1633	if (uHostCr4 & X86_CR4_VMXE)
1634	{
1635	/* Exit VMX root mode and clear the VMX bit in CR4. */
1636	VMXDisable();
1637	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1638	rc = VINF_SUCCESS;
1639	}
1640	else
1641	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1642
1643	/* Restore interrupts. */
1644	ASMSetFlags(fEFlags);
1645	return rc;
1646	}
1647
1648
1649	/**
1650	* Allocates and maps a physically contiguous page. The allocated page is
1651	* zero'd out (used by various VT-x structures).
1652	*
1653	* @returns IPRT status code.
1654	* @param pMemObj Pointer to the ring-0 memory object.
1655	* @param ppVirt Where to store the virtual address of the allocation.
1656	* @param pHCPhys Where to store the physical address of the allocation.
1657	*/
1658	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1659	{
1660	AssertPtr(pMemObj);
1661	AssertPtr(ppVirt);
1662	AssertPtr(pHCPhys);
1663	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1664	if (RT_FAILURE(rc))
1665	return rc;
1666	ppVirt = RTR0MemObjAddress(pMemObj);
1667	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1668	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1669	return VINF_SUCCESS;
1670	}
1671
1672
1673	/**
1674	* Frees and unmaps an allocated, physical page.
1675	*
1676	* @param pMemObj Pointer to the ring-0 memory object.
1677	* @param ppVirt Where to re-initialize the virtual address of allocation as
1678	* 0.
1679	* @param pHCPhys Where to re-initialize the physical address of the
1680	* allocation as 0.
1681	*/
1682	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1683	{
1684	AssertPtr(pMemObj);
1685	AssertPtr(ppVirt);
1686	AssertPtr(pHCPhys);
1687	/* NULL is valid, accepted and ignored by the free function below. */
1688	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1689	*pMemObj = NIL_RTR0MEMOBJ;
1690	*ppVirt = NULL;
1691	*pHCPhys = NIL_RTHCPHYS;
1692	}
1693
1694
1695	/**
1696	* Initializes a VMCS info. object.
1697	*
1698	* @param pVmcsInfo The VMCS info. object.
1699	*/
1700	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1701	{
1702	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1703
1704	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1705	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1706	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1707	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1708	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1709	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1710	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1711	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1712	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1713	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1714	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1715	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1716	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1717	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1718	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1719	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1720	}
1721
1722
1723	/**
1724	* Frees the VT-x structures for a VMCS info. object.
1725	*
1726	* @param pVM The cross context VM structure.
1727	* @param pVmcsInfo The VMCS info. object.
1728	*/
1729	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1730	{
1731	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1732
1733	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1734	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1735	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1736	#endif
1737
1738	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1739	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1740
1741	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1742	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1743	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1744
1745	hmR0VmxInitVmcsInfo(pVmcsInfo);
1746	}
1747
1748
1749	/**
1750	* Allocates the VT-x structures for a VMCS info. object.
1751	*
1752	* @returns VBox status code.
1753	* @param pVCpu The cross context virtual CPU structure.
1754	* @param pVmcsInfo The VMCS info. object.
1755	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1756	*/
1757	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1758	{
1759	PVM pVM = pVCpu->CTX_SUFF(pVM);
1760
1761	/* Allocate the guest VM control structure (VMCS). */
1762	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1763	if (RT_SUCCESS(rc))
1764	{
1765	if (!fIsNstGstVmcs)
1766	{
1767	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1768	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1769	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1770	#endif
1771	if (RT_SUCCESS(rc))
1772	{
1773	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1774	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1775	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1776	{
1777	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1778	NULL /* pR3Ptr /, NULL / pRCPtr */);
1779	}
1780	}
1781	}
1782	else
1783	{
1784	/* We don't yet support exposing VMCS shadowing to the guest. */
1785	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1786	Assert(!pVmcsInfo->pvShadowVmcs);
1787
1788	/* Get the allocated virtual-APIC page from CPUM. */
1789	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1790	{
1791	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1792	&pVmcsInfo->HCPhysVirtApic);
1793	Assert(pVmcsInfo->pbVirtApic);
1794	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1795	}
1796	}
1797
1798	if (RT_SUCCESS(rc))
1799	{
1800	/*
1801	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1802	* transparent accesses of specific MSRs.
1803	*
1804	* If the condition for enabling MSR bitmaps changes here, don't forget to
1805	* update HMIsMsrBitmapActive().
1806	*
1807	* We don't share MSR bitmaps between the guest and nested-guest as we then
1808	* don't need to care about carefully restoring the guest MSR bitmap.
1809	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1810	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1811	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1812	* we do that later while merging VMCS.
1813	*/
1814	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1815	{
1816	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1817	if ( RT_SUCCESS(rc)
1818	&& !fIsNstGstVmcs)
1819	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1820	}
1821
1822	if (RT_SUCCESS(rc))
1823	{
1824	/*
1825	* Allocate the VM-entry MSR-load area for the guest MSRs.
1826	*
1827	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1828	* the guest and nested-guest.
1829	*/
1830	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1831	&pVmcsInfo->HCPhysGuestMsrLoad);
1832	if (RT_SUCCESS(rc))
1833	{
1834	/*
1835	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1836	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1837	*/
1838	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1839	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1840	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1841
1842	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1843	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1844	&pVmcsInfo->HCPhysHostMsrLoad);
1845	}
1846	}
1847	}
1848	}
1849
1850	return rc;
1851	}
1852
1853
1854	/**
1855	* Free all VT-x structures for the VM.
1856	*
1857	* @returns IPRT status code.
1858	* @param pVM The cross context VM structure.
1859	*/
1860	static void hmR0VmxStructsFree(PVM pVM)
1861	{
1862	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1863	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1864	#endif
1865	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1866
1867	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1868	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1869	{
1870	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1871	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1872	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1873	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1874	}
1875	#endif
1876
1877	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1878	{
1879	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1880	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1881	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1882	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1883	if (pVM->cpum.ro.GuestFeatures.fVmx)
1884	{
1885	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1886	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1887	}
1888	#endif
1889	}
1890	}
1891
1892
1893	/**
1894	* Allocate all VT-x structures for the VM.
1895	*
1896	* @returns IPRT status code.
1897	* @param pVM The cross context VM structure.
1898	*/
1899	static int hmR0VmxStructsAlloc(PVM pVM)
1900	{
1901	/*
1902	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1903	* The VMCS size cannot be more than 4096 bytes.
1904	*
1905	* See Intel spec. Appendix A.1 "Basic VMX Information".
1906	*/
1907	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1908	if (cbVmcs <= X86_PAGE_4K_SIZE)
1909	{ /* likely */ }
1910	else
1911	{
1912	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1913	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1914	}
1915
1916	/*
1917	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1918	*/
1919	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1920	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1921	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1922	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1923	#endif
1924
1925	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1926	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1927	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1928
1929	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1930	{
1931	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1932	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1933	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1934	}
1935
1936	/*
1937	* Allocate per-VM VT-x structures.
1938	*/
1939	int rc = VINF_SUCCESS;
1940	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1941	/* Allocate crash-dump magic scratch page. */
1942	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1943	if (RT_FAILURE(rc))
1944	{
1945	hmR0VmxStructsFree(pVM);
1946	return rc;
1947	}
1948	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1949	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1950	#endif
1951
1952	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1953	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1954	{
1955	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1956	&pVM->hm.s.vmx.HCPhysApicAccess);
1957	if (RT_FAILURE(rc))
1958	{
1959	hmR0VmxStructsFree(pVM);
1960	return rc;
1961	}
1962	}
1963
1964	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1965	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1966	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1967	{
1968	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1969	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1970	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1971	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1972	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1973	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1974	{
1975	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1976	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1977	if (RT_SUCCESS(rc))
1978	{
1979	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1980	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1981	}
1982	}
1983	else
1984	rc = VERR_NO_MEMORY;
1985
1986	if (RT_FAILURE(rc))
1987	{
1988	hmR0VmxStructsFree(pVM);
1989	return rc;
1990	}
1991	}
1992	#endif
1993
1994	/*
1995	* Initialize per-VCPU VT-x structures.
1996	*/
1997	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1998	{
1999	/* Allocate the guest VMCS structures. */
2000	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2001	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2002	if (RT_SUCCESS(rc))
2003	{
2004	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2005	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2006	if (pVM->cpum.ro.GuestFeatures.fVmx)
2007	{
2008	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2009	if (RT_SUCCESS(rc))
2010	{ /* likely */ }
2011	else
2012	break;
2013	}
2014	#endif
2015	}
2016	else
2017	break;
2018	}
2019
2020	if (RT_FAILURE(rc))
2021	{
2022	hmR0VmxStructsFree(pVM);
2023	return rc;
2024	}
2025
2026	return VINF_SUCCESS;
2027	}
2028
2029	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2030	/**
2031	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2032	*
2033	* @returns @c true if the MSR is intercepted, @c false otherwise.
2034	* @param pvMsrBitmap The MSR bitmap.
2035	* @param offMsr The MSR byte offset.
2036	* @param iBit The bit offset from the byte offset.
2037	*/
2038	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2039	{
2040	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2041	Assert(pbMsrBitmap);
2042	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2043	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2044	}
2045	#endif
2046
2047	/**
2048	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2049	*
2050	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2051	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2052	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2053	* the read/write access of this MSR.
2054	*
2055	* @param pVCpu The cross context virtual CPU structure.
2056	* @param pVmcsInfo The VMCS info. object.
2057	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2058	* @param idMsr The MSR value.
2059	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2060	* include both a read -and- a write permission!
2061	*
2062	* @sa CPUMGetVmxMsrPermission.
2063	* @remarks Can be called with interrupts disabled.
2064	*/
2065	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2066	{
2067	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2068	Assert(pbMsrBitmap);
2069	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2070
2071	/*
2072	* MSR-bitmap Layout:
2073	* Byte index MSR range Interpreted as
2074	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2075	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2076	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2077	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2078	*
2079	* A bit corresponding to an MSR within the above range causes a VM-exit
2080	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2081	* the MSR range, it always cause a VM-exit.
2082	*
2083	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2084	*/
2085	uint16_t const offBitmapRead = 0;
2086	uint16_t const offBitmapWrite = 0x800;
2087	uint16_t offMsr;
2088	int32_t iBit;
2089	if (idMsr <= UINT32_C(0x00001fff))
2090	{
2091	offMsr = 0;
2092	iBit = idMsr;
2093	}
2094	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2095	{
2096	offMsr = 0x400;
2097	iBit = idMsr - UINT32_C(0xc0000000);
2098	}
2099	else
2100	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2101
2102	/*
2103	* Set the MSR read permission.
2104	*/
2105	uint16_t const offMsrRead = offBitmapRead + offMsr;
2106	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2107	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2108	{
2109	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2110	bool const fClear = !fIsNstGstVmcs ? true
2111	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2112	#else
2113	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2114	bool const fClear = true;
2115	#endif
2116	if (fClear)
2117	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2118	}
2119	else
2120	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2121
2122	/*
2123	* Set the MSR write permission.
2124	*/
2125	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2126	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2127	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2128	{
2129	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2130	bool const fClear = !fIsNstGstVmcs ? true
2131	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2132	#else
2133	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2134	bool const fClear = true;
2135	#endif
2136	if (fClear)
2137	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2138	}
2139	else
2140	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2141	}
2142
2143
2144	/**
2145	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2146	* area.
2147	*
2148	* @returns VBox status code.
2149	* @param pVCpu The cross context virtual CPU structure.
2150	* @param pVmcsInfo The VMCS info. object.
2151	* @param cMsrs The number of MSRs.
2152	*/
2153	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2154	{
2155	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2156	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2157	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2158	{
2159	/* Commit the MSR counts to the VMCS and update the cache. */
2160	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2161	{
2162	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2163	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2164	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2165	AssertRCReturn(rc, rc);
2166
2167	pVmcsInfo->cEntryMsrLoad = cMsrs;
2168	pVmcsInfo->cExitMsrStore = cMsrs;
2169	pVmcsInfo->cExitMsrLoad = cMsrs;
2170	}
2171	return VINF_SUCCESS;
2172	}
2173
2174	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2175	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2176	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2177	}
2178
2179
2180	/**
2181	* Adds a new (or updates the value of an existing) guest/host MSR
2182	* pair to be swapped during the world-switch as part of the
2183	* auto-load/store MSR area in the VMCS.
2184	*
2185	* @returns VBox status code.
2186	* @param pVCpu The cross context virtual CPU structure.
2187	* @param pVmxTransient The VMX-transient structure.
2188	* @param idMsr The MSR.
2189	* @param uGuestMsrValue Value of the guest MSR.
2190	* @param fSetReadWrite Whether to set the guest read/write access of this
2191	* MSR (thus not causing a VM-exit).
2192	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2193	* necessary.
2194	*/
2195	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2196	bool fSetReadWrite, bool fUpdateHostMsr)
2197	{
2198	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2199	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2200	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2201	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2202	uint32_t i;
2203
2204	/* Paranoia. */
2205	Assert(pGuestMsrLoad);
2206
2207	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2208
2209	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2210	for (i = 0; i < cMsrs; i++)
2211	{
2212	if (pGuestMsrLoad[i].u32Msr == idMsr)
2213	break;
2214	}
2215
2216	bool fAdded = false;
2217	if (i == cMsrs)
2218	{
2219	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2220	++cMsrs;
2221	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2222	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2223
2224	/* Set the guest to read/write this MSR without causing VM-exits. */
2225	if ( fSetReadWrite
2226	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2227	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2228
2229	LogFlowFunc(("MSR added, cMsrs now %u\n", cMsrs));
2230	fAdded = true;
2231	}
2232
2233	/* Update the MSR value for the newly added or already existing MSR. */
2234	pGuestMsrLoad[i].u32Msr = idMsr;
2235	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2236
2237	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2238	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2239	{
2240	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2241	pGuestMsrStore[i].u32Msr = idMsr;
2242	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2243	}
2244
2245	/* Update the corresponding slot in the host MSR area. */
2246	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2247	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2248	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2249	pHostMsr[i].u32Msr = idMsr;
2250
2251	/*
2252	* Only if the caller requests to update the host MSR value AND we've newly added the
2253	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2254	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2255	*
2256	* We do this for performance reasons since reading MSRs may be quite expensive.
2257	*/
2258	if (fAdded)
2259	{
2260	if (fUpdateHostMsr)
2261	{
2262	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2263	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2264	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2265	}
2266	else
2267	{
2268	/* Someone else can do the work. */
2269	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2270	}
2271	}
2272	return VINF_SUCCESS;
2273	}
2274
2275
2276	/**
2277	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2278	* auto-load/store MSR area in the VMCS.
2279	*
2280	* @returns VBox status code.
2281	* @param pVCpu The cross context virtual CPU structure.
2282	* @param pVmxTransient The VMX-transient structure.
2283	* @param idMsr The MSR.
2284	*/
2285	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2286	{
2287	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2288	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2289	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2290	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2291
2292	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2293
2294	for (uint32_t i = 0; i < cMsrs; i++)
2295	{
2296	/* Find the MSR. */
2297	if (pGuestMsrLoad[i].u32Msr == idMsr)
2298	{
2299	/*
2300	* If it's the last MSR, we only need to reduce the MSR count.
2301	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2302	*/
2303	if (i < cMsrs - 1)
2304	{
2305	/* Remove it from the VM-entry MSR-load area. */
2306	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2307	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2308
2309	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2310	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2311	{
2312	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2313	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2314	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2315	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2316	}
2317
2318	/* Remove it from the VM-exit MSR-load area. */
2319	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2320	Assert(pHostMsr[i].u32Msr == idMsr);
2321	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2322	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2323	}
2324
2325	/* Reduce the count to reflect the removed MSR and bail. */
2326	--cMsrs;
2327	break;
2328	}
2329	}
2330
2331	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2332	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2333	{
2334	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2335	AssertRCReturn(rc, rc);
2336
2337	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2338	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2339	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2340
2341	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2342	return VINF_SUCCESS;
2343	}
2344
2345	return VERR_NOT_FOUND;
2346	}
2347
2348
2349	/**
2350	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2351	*
2352	* @returns @c true if found, @c false otherwise.
2353	* @param pVmcsInfo The VMCS info. object.
2354	* @param idMsr The MSR to find.
2355	*/
2356	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2357	{
2358	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2359	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2360	Assert(pMsrs);
2361	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2362	for (uint32_t i = 0; i < cMsrs; i++)
2363	{
2364	if (pMsrs[i].u32Msr == idMsr)
2365	return true;
2366	}
2367	return false;
2368	}
2369
2370
2371	/**
2372	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2373	*
2374	* @param pVCpu The cross context virtual CPU structure.
2375	* @param pVmcsInfo The VMCS info. object.
2376	*
2377	* @remarks No-long-jump zone!!!
2378	*/
2379	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2380	{
2381	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2382
2383	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2384	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2385	Assert(pHostMsrLoad);
2386	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2387	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2388	for (uint32_t i = 0; i < cMsrs; i++)
2389	{
2390	/*
2391	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2392	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2393	*/
2394	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2395	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2396	else
2397	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2398	}
2399	}
2400
2401
2402	/**
2403	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2404	* perform lazy restoration of the host MSRs while leaving VT-x.
2405	*
2406	* @param pVCpu The cross context virtual CPU structure.
2407	*
2408	* @remarks No-long-jump zone!!!
2409	*/
2410	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2411	{
2412	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2413
2414	/*
2415	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2416	*/
2417	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2418	{
2419	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2420	#if HC_ARCH_BITS == 64
2421	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2422	{
2423	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2424	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2425	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2426	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2427	}
2428	#endif
2429	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2430	}
2431	}
2432
2433
2434	/**
2435	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2436	* lazily while leaving VT-x.
2437	*
2438	* @returns true if it does, false otherwise.
2439	* @param pVCpu The cross context virtual CPU structure.
2440	* @param idMsr The MSR to check.
2441	*/
2442	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2443	{
2444	NOREF(pVCpu);
2445	#if HC_ARCH_BITS == 64
2446	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2447	{
2448	switch (idMsr)
2449	{
2450	case MSR_K8_LSTAR:
2451	case MSR_K6_STAR:
2452	case MSR_K8_SF_MASK:
2453	case MSR_K8_KERNEL_GS_BASE:
2454	return true;
2455	}
2456	}
2457	#else
2458	RT_NOREF(pVCpu, idMsr);
2459	#endif
2460	return false;
2461	}
2462
2463
2464	/**
2465	* Loads a set of guests MSRs to allow read/passthru to the guest.
2466	*
2467	* The name of this function is slightly confusing. This function does NOT
2468	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2469	* common prefix for functions dealing with "lazy restoration" of the shared
2470	* MSRs.
2471	*
2472	* @param pVCpu The cross context virtual CPU structure.
2473	*
2474	* @remarks No-long-jump zone!!!
2475	*/
2476	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2477	{
2478	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2479	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2480
2481	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2482	#if HC_ARCH_BITS == 64
2483	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2484	{
2485	/*
2486	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2487	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2488	* we can skip a few MSR writes.
2489	*
2490	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2491	* guest MSR values in the guest-CPU context might be different to what's currently
2492	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2493	* CPU, see @bugref{8728}.
2494	*/
2495	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2496	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2497	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2498	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2499	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2500	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2501	{
2502	#ifdef VBOX_STRICT
2503	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2504	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2505	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2506	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2507	#endif
2508	}
2509	else
2510	{
2511	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2512	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2513	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2514	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2515	}
2516	}
2517	#endif
2518	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2519	}
2520
2521
2522	/**
2523	* Performs lazy restoration of the set of host MSRs if they were previously
2524	* loaded with guest MSR values.
2525	*
2526	* @param pVCpu The cross context virtual CPU structure.
2527	*
2528	* @remarks No-long-jump zone!!!
2529	* @remarks The guest MSRs should have been saved back into the guest-CPU
2530	* context by hmR0VmxImportGuestState()!!!
2531	*/
2532	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2533	{
2534	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2535	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2536
2537	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2538	{
2539	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2540	#if HC_ARCH_BITS == 64
2541	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2542	{
2543	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2544	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2545	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2546	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2547	}
2548	#endif
2549	}
2550	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2551	}
2552
2553
2554	/**
2555	* Verifies that our cached values of the VMCS fields are all consistent with
2556	* what's actually present in the VMCS.
2557	*
2558	* @returns VBox status code.
2559	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2560	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2561	* VMCS content. HMCPU error-field is
2562	* updated, see VMX_VCI_XXX.
2563	* @param pVCpu The cross context virtual CPU structure.
2564	* @param pVmcsInfo The VMCS info. object.
2565	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2566	*/
2567	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2568	{
2569	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2570
2571	uint32_t u32Val;
2572	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2573	AssertRCReturn(rc, rc);
2574	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2575	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2576	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2577	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2578
2579	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2580	AssertRCReturn(rc, rc);
2581	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2582	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2583	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2584	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2585
2586	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2587	AssertRCReturn(rc, rc);
2588	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2589	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2590	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2591	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2592
2593	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2594	AssertRCReturn(rc, rc);
2595	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2596	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2597	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2598	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2599
2600	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2601	{
2602	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2603	AssertRCReturn(rc, rc);
2604	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2605	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2606	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2607	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2608	}
2609
2610	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2611	AssertRCReturn(rc, rc);
2612	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2613	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2614	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2615	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2616
2617	uint64_t u64Val;
2618	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2619	AssertRCReturn(rc, rc);
2620	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2621	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2622	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2623	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2624
2625	NOREF(pcszVmcs);
2626	return VINF_SUCCESS;
2627	}
2628
2629
2630	#ifdef VBOX_STRICT
2631	/**
2632	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2633	*
2634	* @param pVCpu The cross context virtual CPU structure.
2635	* @param pVmcsInfo The VMCS info. object.
2636	*/
2637	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2638	{
2639	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2640
2641	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2642	{
2643	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2644	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2645	uint64_t uVmcsEferMsrVmcs;
2646	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2647	AssertRC(rc);
2648
2649	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2650	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2651	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2652	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2653	}
2654	}
2655
2656
2657	/**
2658	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2659	* VMCS are correct.
2660	*
2661	* @param pVCpu The cross context virtual CPU structure.
2662	* @param pVmcsInfo The VMCS info. object.
2663	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2664	*/
2665	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2666	{
2667	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2668
2669	/* Read the various MSR-area counts from the VMCS. */
2670	uint32_t cEntryLoadMsrs;
2671	uint32_t cExitStoreMsrs;
2672	uint32_t cExitLoadMsrs;
2673	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2676
2677	/* Verify all the MSR counts are the same. */
2678	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2679	Assert(cExitStoreMsrs == cExitLoadMsrs);
2680	uint32_t const cMsrs = cExitLoadMsrs;
2681
2682	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2683	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2684
2685	/* Verify the MSR counts are within the allocated page size. */
2686	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2687
2688	/* Verify the relevant contents of the MSR areas match. */
2689	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2690	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2691	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2692	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2693	for (uint32_t i = 0; i < cMsrs; i++)
2694	{
2695	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2696	if (fSeparateExitMsrStorePage)
2697	{
2698	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2699	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2700	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2701	}
2702
2703	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2704	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2705	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2706
2707	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2708	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2709	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2710	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2711
2712	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2713	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2714	if (fIsEferMsr)
2715	{
2716	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2717	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2718	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2719	}
2720
2721	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2722	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2723	{
2724	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2725	if (fIsEferMsr)
2726	{
2727	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2728	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2729	}
2730	else
2731	{
2732	if (!fIsNstGstVmcs)
2733	{
2734	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2735	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2736	}
2737	else
2738	{
2739	/*
2740	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2741	* execute a nested-guest with MSR passthrough.
2742	*
2743	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2744	* allow passthrough too.
2745	*/
2746	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2747	Assert(pvMsrBitmapNstGst);
2748	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2749	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2750	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2751	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2752	}
2753	}
2754	}
2755
2756	/* Move to the next MSR. */
2757	pHostMsrLoad++;
2758	pGuestMsrLoad++;
2759	pGuestMsrStore++;
2760	}
2761	}
2762	#endif /* VBOX_STRICT */
2763
2764
2765	/**
2766	* Flushes the TLB using EPT.
2767	*
2768	* @returns VBox status code.
2769	* @param pVCpu The cross context virtual CPU structure of the calling
2770	* EMT. Can be NULL depending on @a enmTlbFlush.
2771	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2772	* enmTlbFlush.
2773	* @param enmTlbFlush Type of flush.
2774	*
2775	* @remarks Caller is responsible for making sure this function is called only
2776	* when NestedPaging is supported and providing @a enmTlbFlush that is
2777	* supported by the CPU.
2778	* @remarks Can be called with interrupts disabled.
2779	*/
2780	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2781	{
2782	uint64_t au64Descriptor[2];
2783	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2784	au64Descriptor[0] = 0;
2785	else
2786	{
2787	Assert(pVCpu);
2788	Assert(pVmcsInfo);
2789	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2790	}
2791	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2792
2793	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2794	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2795
2796	if ( RT_SUCCESS(rc)
2797	&& pVCpu)
2798	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2799	}
2800
2801
2802	/**
2803	* Flushes the TLB using VPID.
2804	*
2805	* @returns VBox status code.
2806	* @param pVCpu The cross context virtual CPU structure of the calling
2807	* EMT. Can be NULL depending on @a enmTlbFlush.
2808	* @param enmTlbFlush Type of flush.
2809	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2810	* on @a enmTlbFlush).
2811	*
2812	* @remarks Can be called with interrupts disabled.
2813	*/
2814	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2815	{
2816	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2817
2818	uint64_t au64Descriptor[2];
2819	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2820	{
2821	au64Descriptor[0] = 0;
2822	au64Descriptor[1] = 0;
2823	}
2824	else
2825	{
2826	AssertPtr(pVCpu);
2827	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2828	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2829	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2830	au64Descriptor[1] = GCPtr;
2831	}
2832
2833	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2834	AssertMsg(rc == VINF_SUCCESS,
2835	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2836
2837	if ( RT_SUCCESS(rc)
2838	&& pVCpu)
2839	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2840	NOREF(rc);
2841	}
2842
2843
2844	/**
2845	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2846	* otherwise there is nothing really to invalidate.
2847	*
2848	* @returns VBox status code.
2849	* @param pVCpu The cross context virtual CPU structure.
2850	* @param GCVirt Guest virtual address of the page to invalidate.
2851	*/
2852	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2853	{
2854	AssertPtr(pVCpu);
2855	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2856
2857	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2858	{
2859	/*
2860	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2861	* the EPT case. See @bugref{6043} and @bugref{6177}.
2862	*
2863	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2864	* as this function maybe called in a loop with individual addresses.
2865	*/
2866	PVM pVM = pVCpu->CTX_SUFF(pVM);
2867	if (pVM->hm.s.vmx.fVpid)
2868	{
2869	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2870
2871	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2872	/*
2873	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2874	* where executing INVVPID outside 64-bit mode does not flush translations of
2875	* 64-bit linear addresses, see @bugref{6208#c72}.
2876	*/
2877	if (RT_HI_U32(GCVirt))
2878	fVpidFlush = false;
2879	#endif
2880
2881	if (fVpidFlush)
2882	{
2883	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2884	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2885	}
2886	else
2887	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2888	}
2889	else if (pVM->hm.s.fNestedPaging)
2890	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2891	}
2892
2893	return VINF_SUCCESS;
2894	}
2895
2896
2897	/**
2898	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2899	* case where neither EPT nor VPID is supported by the CPU.
2900	*
2901	* @param pHostCpu The HM physical-CPU structure.
2902	* @param pVCpu The cross context virtual CPU structure.
2903	*
2904	* @remarks Called with interrupts disabled.
2905	*/
2906	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2907	{
2908	AssertPtr(pVCpu);
2909	AssertPtr(pHostCpu);
2910
2911	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2912
2913	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2914	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2915	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2916	pVCpu->hm.s.fForceTLBFlush = false;
2917	return;
2918	}
2919
2920
2921	/**
2922	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2923	*
2924	* @param pHostCpu The HM physical-CPU structure.
2925	* @param pVCpu The cross context virtual CPU structure.
2926	* @param pVmcsInfo The VMCS info. object.
2927	*
2928	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2929	* nomenclature. The reason is, to avoid confusion in compare statements
2930	* since the host-CPU copies are named "ASID".
2931	*
2932	* @remarks Called with interrupts disabled.
2933	*/
2934	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2935	{
2936	#ifdef VBOX_WITH_STATISTICS
2937	bool fTlbFlushed = false;
2938	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2939	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2940	if (!fTlbFlushed) \
2941	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2942	} while (0)
2943	#else
2944	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2945	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2946	#endif
2947
2948	AssertPtr(pVCpu);
2949	AssertPtr(pHostCpu);
2950	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2951
2952	PVM pVM = pVCpu->CTX_SUFF(pVM);
2953	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2954	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2955	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2956
2957	/*
2958	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2959	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2960	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2961	* cannot reuse the current ASID anymore.
2962	*/
2963	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2964	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2965	{
2966	++pHostCpu->uCurrentAsid;
2967	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2968	{
2969	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2970	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2971	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2972	}
2973
2974	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2975	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2976	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2977
2978	/*
2979	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2980	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2981	*/
2982	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2983	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2984	HMVMX_SET_TAGGED_TLB_FLUSHED();
2985	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2986	}
2987	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2988	{
2989	/*
2990	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2991	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2992	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2993	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2994	* mappings, see @bugref{6568}.
2995	*
2996	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2997	*/
2998	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2999	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3000	HMVMX_SET_TAGGED_TLB_FLUSHED();
3001	}
3002
3003	pVCpu->hm.s.fForceTLBFlush = false;
3004	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3005
3006	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3007	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3008	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3009	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3010	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3011	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3012	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3013	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3014	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3015
3016	/* Update VMCS with the VPID. */
3017	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3018	AssertRC(rc);
3019
3020	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3021	}
3022
3023
3024	/**
3025	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3026	*
3027	* @param pHostCpu The HM physical-CPU structure.
3028	* @param pVCpu The cross context virtual CPU structure.
3029	* @param pVmcsInfo The VMCS info. object.
3030	*
3031	* @remarks Called with interrupts disabled.
3032	*/
3033	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3034	{
3035	AssertPtr(pVCpu);
3036	AssertPtr(pHostCpu);
3037	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3038	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3039	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3040
3041	/*
3042	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3043	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3044	*/
3045	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3046	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3047	{
3048	pVCpu->hm.s.fForceTLBFlush = true;
3049	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3050	}
3051
3052	/* Check for explicit TLB flushes. */
3053	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3054	{
3055	pVCpu->hm.s.fForceTLBFlush = true;
3056	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3057	}
3058
3059	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3060	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3061
3062	if (pVCpu->hm.s.fForceTLBFlush)
3063	{
3064	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3065	pVCpu->hm.s.fForceTLBFlush = false;
3066	}
3067	}
3068
3069
3070	/**
3071	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3072	*
3073	* @param pHostCpu The HM physical-CPU structure.
3074	* @param pVCpu The cross context virtual CPU structure.
3075	*
3076	* @remarks Called with interrupts disabled.
3077	*/
3078	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3079	{
3080	AssertPtr(pVCpu);
3081	AssertPtr(pHostCpu);
3082	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3083	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3084	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3085
3086	/*
3087	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3088	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3089	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3090	* cannot reuse the current ASID anymore.
3091	*/
3092	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3093	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3094	{
3095	pVCpu->hm.s.fForceTLBFlush = true;
3096	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3097	}
3098
3099	/* Check for explicit TLB flushes. */
3100	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3101	{
3102	/*
3103	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3104	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3105	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3106	* include fExplicitFlush's too) - an obscure corner case.
3107	*/
3108	pVCpu->hm.s.fForceTLBFlush = true;
3109	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3110	}
3111
3112	PVM pVM = pVCpu->CTX_SUFF(pVM);
3113	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3114	if (pVCpu->hm.s.fForceTLBFlush)
3115	{
3116	++pHostCpu->uCurrentAsid;
3117	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3118	{
3119	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3120	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3121	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3122	}
3123
3124	pVCpu->hm.s.fForceTLBFlush = false;
3125	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3126	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3127	if (pHostCpu->fFlushAsidBeforeUse)
3128	{
3129	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3130	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3131	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3132	{
3133	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3134	pHostCpu->fFlushAsidBeforeUse = false;
3135	}
3136	else
3137	{
3138	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3139	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3140	}
3141	}
3142	}
3143
3144	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3145	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3146	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3147	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3148	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3149	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3150	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3151
3152	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3153	AssertRC(rc);
3154	}
3155
3156
3157	/**
3158	* Flushes the guest TLB entry based on CPU capabilities.
3159	*
3160	* @param pHostCpu The HM physical-CPU structure.
3161	* @param pVCpu The cross context virtual CPU structure.
3162	* @param pVmcsInfo The VMCS info. object.
3163	*
3164	* @remarks Called with interrupts disabled.
3165	*/
3166	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3167	{
3168	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3169	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3170	#endif
3171	PVM pVM = pVCpu->CTX_SUFF(pVM);
3172	switch (pVM->hm.s.vmx.enmTlbFlushType)
3173	{
3174	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3175	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3176	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3177	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3178	default:
3179	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3180	break;
3181	}
3182	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3183	}
3184
3185
3186	/**
3187	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3188	* TLB entries from the host TLB before VM-entry.
3189	*
3190	* @returns VBox status code.
3191	* @param pVM The cross context VM structure.
3192	*/
3193	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3194	{
3195	/*
3196	* Determine optimal flush type for nested paging.
3197	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3198	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3199	*/
3200	if (pVM->hm.s.fNestedPaging)
3201	{
3202	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3203	{
3204	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3205	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3206	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3207	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3208	else
3209	{
3210	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3211	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3212	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3213	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3214	}
3215
3216	/* Make sure the write-back cacheable memory type for EPT is supported. */
3217	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3218	{
3219	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3220	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3221	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3222	}
3223
3224	/* EPT requires a page-walk length of 4. */
3225	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3226	{
3227	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3228	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3229	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3230	}
3231	}
3232	else
3233	{
3234	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3235	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3236	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3237	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3238	}
3239	}
3240
3241	/*
3242	* Determine optimal flush type for VPID.
3243	*/
3244	if (pVM->hm.s.vmx.fVpid)
3245	{
3246	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3247	{
3248	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3249	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3250	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3251	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3252	else
3253	{
3254	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3255	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3256	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3257	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3258	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3259	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3260	pVM->hm.s.vmx.fVpid = false;
3261	}
3262	}
3263	else
3264	{
3265	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3266	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3267	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3268	pVM->hm.s.vmx.fVpid = false;
3269	}
3270	}
3271
3272	/*
3273	* Setup the handler for flushing tagged-TLBs.
3274	*/
3275	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3276	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3277	else if (pVM->hm.s.fNestedPaging)
3278	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3279	else if (pVM->hm.s.vmx.fVpid)
3280	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3281	else
3282	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3283	return VINF_SUCCESS;
3284	}
3285
3286
3287	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3288	/**
3289	* Sets up the shadow VMCS fields arrays.
3290	*
3291	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3292	* executing the guest.
3293	*
3294	* @returns VBox status code.
3295	* @param pVM The cross context VM structure.
3296	*/
3297	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3298	{
3299	/*
3300	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3301	* when the host does not support it.
3302	*/
3303	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3304	if ( !fGstVmwriteAll
3305	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3306	{ /* likely. */ }
3307	else
3308	{
3309	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3310	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3311	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3312	}
3313
3314	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3315	uint32_t cRwFields = 0;
3316	uint32_t cRoFields = 0;
3317	for (uint32_t i = 0; i < cVmcsFields; i++)
3318	{
3319	VMXVMCSFIELD VmcsField;
3320	VmcsField.u = g_aVmcsFields[i];
3321
3322	/*
3323	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3324	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3325	* in the shadow VMCS fields array as they would be redundant.
3326	*
3327	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3328	* we must not include it in the shadow VMCS fields array. Guests attempting to
3329	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3330	* the required behavior.
3331	*/
3332	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3333	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3334	{
3335	/*
3336	* Read-only fields are placed in a separate array so that while syncing shadow
3337	* VMCS fields later (which is more performance critical) we can avoid branches.
3338	*
3339	* However, if the guest can write to all fields (including read-only fields),
3340	* we treat it a as read/write field. Otherwise, writing to these fields would
3341	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3342	*/
3343	if ( fGstVmwriteAll
3344	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3345	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3346	else
3347	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3348	}
3349	}
3350
3351	/* Update the counts. */
3352	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3353	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3354	return VINF_SUCCESS;
3355	}
3356
3357
3358	/**
3359	* Sets up the VMREAD and VMWRITE bitmaps.
3360	*
3361	* @param pVM The cross context VM structure.
3362	*/
3363	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3364	{
3365	/*
3366	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3367	*/
3368	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3369	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3370	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3371	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3372	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3373
3374	/*
3375	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3376	* VMREAD and VMWRITE bitmaps.
3377	*/
3378	{
3379	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3380	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3381	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3382	{
3383	uint32_t const uVmcsField = paShadowVmcsFields[i];
3384	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3385	Assert(uVmcsField >> 3 < cbBitmap);
3386	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3387	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3388	}
3389	}
3390
3391	/*
3392	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3393	* if the host supports VMWRITE to all supported VMCS fields.
3394	*/
3395	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3396	{
3397	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3398	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3399	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3400	{
3401	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3402	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3403	Assert(uVmcsField >> 3 < cbBitmap);
3404	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3405	}
3406	}
3407	}
3408	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3409
3410
3411	/**
3412	* Sets up the virtual-APIC page address for the VMCS.
3413	*
3414	* @returns VBox status code.
3415	* @param pVCpu The cross context virtual CPU structure.
3416	* @param pVmcsInfo The VMCS info. object.
3417	*/
3418	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3419	{
3420	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3421	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3422	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3423	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3424	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3425	}
3426
3427
3428	/**
3429	* Sets up the MSR-bitmap address for the VMCS.
3430	*
3431	* @returns VBox status code.
3432	* @param pVCpu The cross context virtual CPU structure.
3433	* @param pVmcsInfo The VMCS info. object.
3434	*/
3435	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3436	{
3437	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3438	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3439	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3440	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3441	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3442	}
3443
3444
3445	/**
3446	* Sets up the APIC-access page address for the VMCS.
3447	*
3448	* @returns VBox status code.
3449	* @param pVCpu The cross context virtual CPU structure.
3450	*/
3451	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3452	{
3453	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3454	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3455	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3456	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3457	}
3458
3459
3460	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3461	/**
3462	* Sets up the VMREAD bitmap address for the VMCS.
3463	*
3464	* @returns VBox status code.
3465	* @param pVCpu The cross context virtual CPU structure.
3466	*/
3467	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3468	{
3469	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3470	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3471	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3472	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3473	}
3474
3475
3476	/**
3477	* Sets up the VMWRITE bitmap address for the VMCS.
3478	*
3479	* @returns VBox status code.
3480	* @param pVCpu The cross context virtual CPU structure.
3481	*/
3482	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3483	{
3484	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3485	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3486	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3487	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3488	}
3489	#endif
3490
3491
3492	/**
3493	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3494	* in the VMCS.
3495	*
3496	* @returns VBox status code.
3497	* @param pVCpu The cross context virtual CPU structure.
3498	* @param pVmcsInfo The VMCS info. object.
3499	*/
3500	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3501	{
3502	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3503
3504	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3505	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3506	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3507
3508	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3509	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3510	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3511
3512	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3513	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3514	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3515
3516	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3517	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3518	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3519	AssertRCReturn(rc, rc);
3520	return VINF_SUCCESS;
3521	}
3522
3523
3524	/**
3525	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3526	*
3527	* @param pVCpu The cross context virtual CPU structure.
3528	* @param pVmcsInfo The VMCS info. object.
3529	*/
3530	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3531	{
3532	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3533
3534	/*
3535	* The guest can access the following MSRs (read, write) without causing
3536	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3537	*/
3538	PVM pVM = pVCpu->CTX_SUFF(pVM);
3539	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3540	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3541	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3542	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3543	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3544
3545	/*
3546	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3547	* associated with then. We never need to intercept access (writes need to be
3548	* executed without causing a VM-exit, reads will #GP fault anyway).
3549	*
3550	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3551	* read/write them. We swap the the guest/host MSR value using the
3552	* auto-load/store MSR area.
3553	*/
3554	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3555	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3556	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3557	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3558	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3559	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3560
3561	#if HC_ARCH_BITS == 64
3562	/*
3563	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3564	* required for 64-bit guests.
3565	*/
3566	if (pVM->hm.s.fAllow64BitGuests)
3567	{
3568	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3569	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3570	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3571	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3572	}
3573	#endif
3574
3575	/*
3576	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3577	*/
3578	#ifdef VBOX_STRICT
3579	Assert(pVmcsInfo->pvMsrBitmap);
3580	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3581	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3582	#endif
3583	}
3584
3585
3586	/**
3587	* Sets up pin-based VM-execution controls in the VMCS.
3588	*
3589	* @returns VBox status code.
3590	* @param pVCpu The cross context virtual CPU structure.
3591	* @param pVmcsInfo The VMCS info. object.
3592	*/
3593	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3594	{
3595	PVM pVM = pVCpu->CTX_SUFF(pVM);
3596	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3597	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3598
3599	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3600	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3601
3602	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3603	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3604
3605	/* Enable the VMX-preemption timer. */
3606	if (pVM->hm.s.vmx.fUsePreemptTimer)
3607	{
3608	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3609	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3610	}
3611
3612	#if 0
3613	/* Enable posted-interrupt processing. */
3614	if (pVM->hm.s.fPostedIntrs)
3615	{
3616	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3617	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3618	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3619	}
3620	#endif
3621
3622	if ((fVal & fZap) != fVal)
3623	{
3624	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3625	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3626	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3627	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3628	}
3629
3630	/* Commit it to the VMCS and update our cache. */
3631	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3632	AssertRCReturn(rc, rc);
3633	pVmcsInfo->u32PinCtls = fVal;
3634
3635	return VINF_SUCCESS;
3636	}
3637
3638
3639	/**
3640	* Sets up secondary processor-based VM-execution controls in the VMCS.
3641	*
3642	* @returns VBox status code.
3643	* @param pVCpu The cross context virtual CPU structure.
3644	* @param pVmcsInfo The VMCS info. object.
3645	*/
3646	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3647	{
3648	PVM pVM = pVCpu->CTX_SUFF(pVM);
3649	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3650	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3651
3652	/* WBINVD causes a VM-exit. */
3653	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3654	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3655
3656	/* Enable EPT (aka nested-paging). */
3657	if (pVM->hm.s.fNestedPaging)
3658	fVal \|= VMX_PROC_CTLS2_EPT;
3659
3660	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3661	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3662	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3663	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3664	fVal \|= VMX_PROC_CTLS2_INVPCID;
3665
3666	/* Enable VPID. */
3667	if (pVM->hm.s.vmx.fVpid)
3668	fVal \|= VMX_PROC_CTLS2_VPID;
3669
3670	/* Enable unrestricted guest execution. */
3671	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3672	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3673
3674	#if 0
3675	if (pVM->hm.s.fVirtApicRegs)
3676	{
3677	/* Enable APIC-register virtualization. */
3678	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3679	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3680
3681	/* Enable virtual-interrupt delivery. */
3682	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3683	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3684	}
3685	#endif
3686
3687	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3688	where the TPR shadow resides. */
3689	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3690	* done dynamically. */
3691	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3692	{
3693	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3694	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3695	AssertRCReturn(rc, rc);
3696	}
3697
3698	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3699	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3700	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3701	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3702	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3703
3704	/* Enable Pause-Loop exiting. */
3705	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3706	&& pVM->hm.s.vmx.cPleGapTicks
3707	&& pVM->hm.s.vmx.cPleWindowTicks)
3708	{
3709	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3710
3711	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3712	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3713	AssertRCReturn(rc, rc);
3714	}
3715
3716	if ((fVal & fZap) != fVal)
3717	{
3718	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3719	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3720	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3721	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3722	}
3723
3724	/* Commit it to the VMCS and update our cache. */
3725	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3726	AssertRCReturn(rc, rc);
3727	pVmcsInfo->u32ProcCtls2 = fVal;
3728
3729	return VINF_SUCCESS;
3730	}
3731
3732
3733	/**
3734	* Sets up processor-based VM-execution controls in the VMCS.
3735	*
3736	* @returns VBox status code.
3737	* @param pVCpu The cross context virtual CPU structure.
3738	* @param pVmcsInfo The VMCS info. object.
3739	*/
3740	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3741	{
3742	PVM pVM = pVCpu->CTX_SUFF(pVM);
3743
3744	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3745	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3746
3747	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3748	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3749	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3750	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3751	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3752	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3753	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3754
3755	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3756	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3757	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3758	{
3759	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3760	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3761	}
3762
3763	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3764	if (!pVM->hm.s.fNestedPaging)
3765	{
3766	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3767	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3768	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3769	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3770	}
3771
3772	/* Use TPR shadowing if supported by the CPU. */
3773	if ( PDMHasApic(pVM)
3774	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3775	{
3776	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3777	/* CR8 writes cause a VM-exit based on TPR threshold. */
3778	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3779	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3780	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3781	AssertRCReturn(rc, rc);
3782	}
3783	else
3784	{
3785	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3786	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3787	if (pVM->hm.s.fAllow64BitGuests)
3788	{
3789	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3790	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3791	}
3792	}
3793
3794	/* Use MSR-bitmaps if supported by the CPU. */
3795	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3796	{
3797	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3798	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3799	AssertRCReturn(rc, rc);
3800	}
3801
3802	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3803	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3804	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3805
3806	if ((fVal & fZap) != fVal)
3807	{
3808	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3809	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3810	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3811	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3812	}
3813
3814	/* Commit it to the VMCS and update our cache. */
3815	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3816	AssertRCReturn(rc, rc);
3817	pVmcsInfo->u32ProcCtls = fVal;
3818
3819	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3820	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3821	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3822
3823	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3824	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3825	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3826
3827	/* Sanity check, should not really happen. */
3828	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3829	{ /* likely */ }
3830	else
3831	{
3832	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3833	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3834	}
3835
3836	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3837	return VINF_SUCCESS;
3838	}
3839
3840
3841	/**
3842	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3843	* Processor-based VM-execution) control fields in the VMCS.
3844	*
3845	* @returns VBox status code.
3846	* @param pVCpu The cross context virtual CPU structure.
3847	* @param pVmcsInfo The VMCS info. object.
3848	*/
3849	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3850	{
3851	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3852	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3853	{
3854	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3855	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3856	if (RT_SUCCESS(rc))
3857	{ /* likely */ }
3858	else
3859	{
3860	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3861	return rc;
3862	}
3863	}
3864	#endif
3865
3866	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3867	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3868	if (RT_SUCCESS(rc))
3869	{
3870	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3871	if (RT_SUCCESS(rc))
3872	{
3873	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3874	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3875	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3876	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3877	if (RT_SUCCESS(rc))
3878	{
3879	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3880	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3881	return VINF_SUCCESS;
3882	}
3883	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3884	}
3885	else
3886	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3887	}
3888	else
3889	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3890	return rc;
3891	}
3892
3893
3894	/**
3895	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3896	*
3897	* We shall setup those exception intercepts that don't change during the
3898	* lifetime of the VM here. The rest are done dynamically while loading the
3899	* guest state.
3900	*
3901	* @returns VBox status code.
3902	* @param pVCpu The cross context virtual CPU structure.
3903	* @param pVmcsInfo The VMCS info. object.
3904	*/
3905	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3906	{
3907	/*
3908	* The following exceptions are always intercepted:
3909	*
3910	* #AC - To prevent the guest from hanging the CPU.
3911	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3912	* recursive #DBs can cause a CPU hang.
3913	* #PF - To sync our shadow page tables when nested-paging is not used.
3914	*/
3915	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3916	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3917	\| RT_BIT(X86_XCPT_DB)
3918	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3919
3920	/* Commit it to the VMCS. */
3921	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3922	AssertRCReturn(rc, rc);
3923
3924	/* Update our cache of the exception bitmap. */
3925	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3926	return VINF_SUCCESS;
3927	}
3928
3929
3930	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3931	/**
3932	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3933	*
3934	* @returns VBox status code.
3935	* @param pVCpu The cross context virtual CPU structure.
3936	* @param pVmcsInfo The VMCS info. object.
3937	*/
3938	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3939	{
3940	PVM pVM = pVCpu->CTX_SUFF(pVM);
3941	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3942	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3943	if (RT_SUCCESS(rc))
3944	{
3945	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3946	if (RT_SUCCESS(rc))
3947	{
3948	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3949	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3950	if (RT_SUCCESS(rc))
3951	{
3952	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3953	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3954	if (RT_SUCCESS(rc))
3955	return VINF_SUCCESS;
3956
3957	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3958	}
3959	else
3960	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3961	}
3962	else
3963	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3964	}
3965	else
3966	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3967
3968	return rc;
3969	}
3970	#endif
3971
3972
3973	/**
3974	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3975	* VMX.
3976	*
3977	* @returns VBox status code.
3978	* @param pVCpu The cross context virtual CPU structure.
3979	* @param pVmcsInfo The VMCS info. object.
3980	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3981	*/
3982	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3983	{
3984	Assert(pVmcsInfo->pvVmcs);
3985	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3986
3987	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3988	PVM pVM = pVCpu->CTX_SUFF(pVM);
3989	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3990	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3991
3992	LogFlowFunc(("\n"));
3993
3994	/*
3995	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3996	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3997	*/
3998	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3999	if (RT_SUCCESS(rc))
4000	{
4001	rc = hmR0VmxLoadVmcs(pVmcsInfo);
4002	if (RT_SUCCESS(rc))
4003	{
4004	if (!fIsNstGstVmcs)
4005	{
4006	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4007	if (RT_SUCCESS(rc))
4008	{
4009	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4010	if (RT_SUCCESS(rc))
4011	{
4012	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4013	if (RT_SUCCESS(rc))
4014	{
4015	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4016	if (RT_SUCCESS(rc))
4017	{
4018	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4019	/*
4020	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4021	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4022	* making it fit for use when VMCS shadowing is later enabled.
4023	*/
4024	if (pVmcsInfo->pvShadowVmcs)
4025	{
4026	VMXVMCSREVID VmcsRevId;
4027	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4028	VmcsRevId.n.fIsShadowVmcs = 1;
4029	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4030	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4031	if (RT_SUCCESS(rc))
4032	{ /* likely */ }
4033	else
4034	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4035	}
4036	#endif
4037	}
4038	else
4039	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4040	}
4041	else
4042	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4043	}
4044	else
4045	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4046	}
4047	else
4048	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4049	}
4050	else
4051	{
4052	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4053	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4054	if (RT_SUCCESS(rc))
4055	{ /* likely */ }
4056	else
4057	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4058	#else
4059	AssertFailed();
4060	#endif
4061	}
4062	}
4063	else
4064	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4065	}
4066	else
4067	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4068
4069	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4070	if (RT_SUCCESS(rc))
4071	{
4072	rc = hmR0VmxClearVmcs(pVmcsInfo);
4073	if (RT_SUCCESS(rc))
4074	{ /* likely */ }
4075	else
4076	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4077	}
4078
4079	/*
4080	* Update the last-error record both for failures and success, so we
4081	* can propagate the status code back to ring-3 for diagnostics.
4082	*/
4083	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4084	NOREF(pszVmcs);
4085	return rc;
4086	}
4087
4088
4089	/**
4090	* Does global VT-x initialization (called during module initialization).
4091	*
4092	* @returns VBox status code.
4093	*/
4094	VMMR0DECL(int) VMXR0GlobalInit(void)
4095	{
4096	#ifdef HMVMX_USE_FUNCTION_TABLE
4097	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4098	# ifdef VBOX_STRICT
4099	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4100	Assert(g_apfnVMExitHandlers[i]);
4101	# endif
4102	#endif
4103	return VINF_SUCCESS;
4104	}
4105
4106
4107	/**
4108	* Does global VT-x termination (called during module termination).
4109	*/
4110	VMMR0DECL(void) VMXR0GlobalTerm()
4111	{
4112	/* Nothing to do currently. */
4113	}
4114
4115
4116	/**
4117	* Sets up and activates VT-x on the current CPU.
4118	*
4119	* @returns VBox status code.
4120	* @param pHostCpu The HM physical-CPU structure.
4121	* @param pVM The cross context VM structure. Can be
4122	* NULL after a host resume operation.
4123	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4124	* fEnabledByHost is @c true).
4125	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4126	* @a fEnabledByHost is @c true).
4127	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4128	* enable VT-x on the host.
4129	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4130	*/
4131	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4132	PCSUPHWVIRTMSRS pHwvirtMsrs)
4133	{
4134	AssertPtr(pHostCpu);
4135	AssertPtr(pHwvirtMsrs);
4136	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4137
4138	/* Enable VT-x if it's not already enabled by the host. */
4139	if (!fEnabledByHost)
4140	{
4141	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4142	if (RT_FAILURE(rc))
4143	return rc;
4144	}
4145
4146	/*
4147	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4148	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4149	* invalidated when flushing by VPID.
4150	*/
4151	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4152	{
4153	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4154	pHostCpu->fFlushAsidBeforeUse = false;
4155	}
4156	else
4157	pHostCpu->fFlushAsidBeforeUse = true;
4158
4159	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4160	++pHostCpu->cTlbFlushes;
4161
4162	return VINF_SUCCESS;
4163	}
4164
4165
4166	/**
4167	* Deactivates VT-x on the current CPU.
4168	*
4169	* @returns VBox status code.
4170	* @param pvCpuPage Pointer to the VMXON region.
4171	* @param HCPhysCpuPage Physical address of the VMXON region.
4172	*
4173	* @remarks This function should never be called when SUPR0EnableVTx() or
4174	* similar was used to enable VT-x on the host.
4175	*/
4176	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4177	{
4178	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4179
4180	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4181	return hmR0VmxLeaveRootMode();
4182	}
4183
4184
4185	/**
4186	* Does per-VM VT-x initialization.
4187	*
4188	* @returns VBox status code.
4189	* @param pVM The cross context VM structure.
4190	*/
4191	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4192	{
4193	AssertPtr(pVM);
4194	LogFlowFunc(("pVM=%p\n", pVM));
4195
4196	int rc = hmR0VmxStructsAlloc(pVM);
4197	if (RT_FAILURE(rc))
4198	{
4199	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4200	return rc;
4201	}
4202
4203	return VINF_SUCCESS;
4204	}
4205
4206
4207	/**
4208	* Does per-VM VT-x termination.
4209	*
4210	* @returns VBox status code.
4211	* @param pVM The cross context VM structure.
4212	*/
4213	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4214	{
4215	AssertPtr(pVM);
4216	LogFlowFunc(("pVM=%p\n", pVM));
4217
4218	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4219	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4220	{
4221	Assert(pVM->hm.s.vmx.pvScratch);
4222	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4223	}
4224	#endif
4225	hmR0VmxStructsFree(pVM);
4226	return VINF_SUCCESS;
4227	}
4228
4229
4230	/**
4231	* Sets up the VM for execution using hardware-assisted VMX.
4232	* This function is only called once per-VM during initialization.
4233	*
4234	* @returns VBox status code.
4235	* @param pVM The cross context VM structure.
4236	*/
4237	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4238	{
4239	AssertPtr(pVM);
4240	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4241
4242	LogFlowFunc(("pVM=%p\n", pVM));
4243
4244	/*
4245	* At least verify if VMX is enabled, since we can't check if we're in
4246	* VMX root mode or not without causing a #GP.
4247	*/
4248	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4249	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4250	{ /* likely */ }
4251	else
4252	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4253
4254	/*
4255	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4256	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4257	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4258	*/
4259	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4261	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4262	{
4263	LogRelFunc(("Invalid real-on-v86 state.\n"));
4264	return VERR_INTERNAL_ERROR;
4265	}
4266
4267	/* Initialize these always, see hmR3InitFinalizeR0().*/
4268	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4269	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4270
4271	/* Setup the tagged-TLB flush handlers. */
4272	int rc = hmR0VmxSetupTaggedTlb(pVM);
4273	if (RT_FAILURE(rc))
4274	{
4275	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4276	return rc;
4277	}
4278
4279	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4280	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4281	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4282	{
4283	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4284	if (RT_SUCCESS(rc))
4285	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4286	else
4287	{
4288	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4289	return rc;
4290	}
4291	}
4292	#endif
4293
4294	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4295	{
4296	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4297	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4298
4299	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4300	if (RT_SUCCESS(rc))
4301	{
4302	#if HC_ARCH_BITS == 32
4303	hmR0VmxInitVmcsReadCache(pVCpu);
4304	#endif
4305	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4306	if (pVM->cpum.ro.GuestFeatures.fVmx)
4307	{
4308	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4309	if (RT_SUCCESS(rc))
4310	{ /* likely */ }
4311	else
4312	{
4313	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4314	return rc;
4315	}
4316	}
4317	#endif
4318	}
4319	else
4320	{
4321	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4322	return rc;
4323	}
4324	}
4325
4326	return VINF_SUCCESS;
4327	}
4328
4329
4330	#if HC_ARCH_BITS == 32
4331	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4332	/**
4333	* Check if guest state allows safe use of 32-bit switcher again.
4334	*
4335	* Segment bases and protected mode structures must be 32-bit addressable
4336	* because the 32-bit switcher will ignore high dword when writing these VMCS
4337	* fields. See @bugref{8432} for details.
4338	*
4339	* @returns true if safe, false if must continue to use the 64-bit switcher.
4340	* @param pCtx Pointer to the guest-CPU context.
4341	*
4342	* @remarks No-long-jump zone!!!
4343	*/
4344	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4345	{
4346	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4347	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4348	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4349	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4350	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4351	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4352	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4353	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4354	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4355	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4356
4357	/* All good, bases are 32-bit. */
4358	return true;
4359	}
4360	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
4361
4362	# ifdef VBOX_STRICT
4363	static bool hmR0VmxIsValidWriteFieldInCache(uint32_t idxField)
4364	{
4365	switch (idxField)
4366	{
4367	case VMX_VMCS_GUEST_RIP:
4368	case VMX_VMCS_GUEST_RSP:
4369	case VMX_VMCS_GUEST_SYSENTER_EIP:
4370	case VMX_VMCS_GUEST_SYSENTER_ESP:
4371	case VMX_VMCS_GUEST_GDTR_BASE:
4372	case VMX_VMCS_GUEST_IDTR_BASE:
4373	case VMX_VMCS_GUEST_CS_BASE:
4374	case VMX_VMCS_GUEST_DS_BASE:
4375	case VMX_VMCS_GUEST_ES_BASE:
4376	case VMX_VMCS_GUEST_FS_BASE:
4377	case VMX_VMCS_GUEST_GS_BASE:
4378	case VMX_VMCS_GUEST_SS_BASE:
4379	case VMX_VMCS_GUEST_LDTR_BASE:
4380	case VMX_VMCS_GUEST_TR_BASE:
4381	case VMX_VMCS_GUEST_CR3:
4382	return true;
4383	}
4384	return false;
4385	}
4386
4387	static bool hmR0VmxIsValidReadFieldInCache(uint32_t idxField)
4388	{
4389	switch (idxField)
4390	{
4391	/* Read-only fields. */
4392	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4393	return true;
4394	}
4395	/* Remaining readable fields should also be writable. */
4396	return hmR0VmxIsValidWriteFieldInCache(idxField);
4397	}
4398	# endif /* VBOX_STRICT */
4399
4400
4401	/**
4402	* Executes the specified handler in 64-bit mode.
4403	*
4404	* @returns VBox status code (no informational status codes).
4405	* @param pVCpu The cross context virtual CPU structure.
4406	* @param enmOp The operation to perform.
4407	* @param cParams Number of parameters.
4408	* @param paParam Array of 32-bit parameters.
4409	*/
4410	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
4411	{
4412	AssertPtr(pVCpu);
4413	PVM pVM = pVCpu->CTX_SUFF(pVM);
4414	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4415	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4416	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
4417	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
4418
4419	#ifdef VBOX_STRICT
4420	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
4421	Assert(hmR0VmxIsValidWriteFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
4422
4423	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
4424	Assert(hmR0VmxIsValidReadFieldInCache(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
4425	#endif
4426
4427	/* Disable interrupts. */
4428	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
4429
4430	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4431	RTCPUID idHostCpu = RTMpCpuId();
4432	CPUMR0SetLApic(pVCpu, idHostCpu);
4433	#endif
4434
4435	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4436
4437	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4438	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4439
4440	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4441	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4442	hmR0VmxClearVmcs(pVmcsInfo);
4443
4444	/* Leave VMX root mode and disable VMX. */
4445	VMXDisable();
4446	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4447
4448	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4449	CPUMSetHyperEIP(pVCpu, enmOp);
4450	for (int i = (int)cParams - 1; i >= 0; i--)
4451	CPUMPushHyper(pVCpu, paParam[i]);
4452
4453	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4454
4455	/* Call the switcher. */
4456	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
4457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4458
4459	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
4460	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
4461
4462	/* Re-enter VMX root mode. */
4463	int rc2 = VMXEnable(HCPhysCpuPage);
4464	if (RT_FAILURE(rc2))
4465	{
4466	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
4467	ASMSetFlags(fOldEFlags);
4468	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
4469	return rc2;
4470	}
4471
4472	/* Restore the VMCS as the current VMCS. */
4473	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
4474	AssertRC(rc2);
4475	Assert(!(ASMGetFlags() & X86_EFL_IF));
4476	ASMSetFlags(fOldEFlags);
4477	return rc;
4478	}
4479
4480
4481	/**
4482	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4483	* supporting 64-bit guests.
4484	*
4485	* @returns VBox status code.
4486	* @param fResume Whether to VMLAUNCH or VMRESUME.
4487	* @param pCtx Pointer to the guest-CPU context.
4488	* @param pCache Pointer to the VMCS batch cache.
4489	* @param pVM The cross context VM structure.
4490	* @param pVCpu The cross context virtual CPU structure.
4491	*/
4492	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4493	{
4494	NOREF(fResume);
4495
4496	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
4497	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4498	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
4499
4500	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4501	pCache->uPos = 1;
4502	pCache->interPD = PGMGetInterPaeCR3(pVM);
4503	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4504	#endif
4505
4506	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4507	pCache->TestIn.HCPhysCpuPage = 0;
4508	pCache->TestIn.HCPhysVmcs = 0;
4509	pCache->TestIn.pCache = 0;
4510	pCache->TestOut.HCPhysVmcs = 0;
4511	pCache->TestOut.pCache = 0;
4512	pCache->TestOut.pCtx = 0;
4513	pCache->TestOut.eflags = 0;
4514	#else
4515	NOREF(pCache);
4516	#endif
4517
4518	uint32_t aParam[10];
4519	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4520	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
4521	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4522	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
4523	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
4524	aParam[5] = 0;
4525	aParam[6] = VM_RC_ADDR(pVM, pVM);
4526	aParam[7] = 0;
4527	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
4528	aParam[9] = 0;
4529
4530	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4531	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4532	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4533	#endif
4534	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
4535
4536	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4537	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4538	Assert(pCtx->dr[4] == 10);
4539	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4540	#endif
4541
4542	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4543	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4544	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4545	pVmcsInfo->HCPhysVmcs));
4546	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4547	pCache->TestOut.HCPhysVmcs));
4548	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4549	pCache->TestOut.pCache));
4550	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
4551	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
4552	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4553	pCache->TestOut.pCtx));
4554	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4555	#endif
4556	NOREF(pCtx);
4557	return rc;
4558	}
4559	#endif
4560
4561
4562	/**
4563	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4564	* the VMCS.
4565	*
4566	* @returns VBox status code.
4567	*/
4568	static int hmR0VmxExportHostControlRegs(void)
4569	{
4570	RTCCUINTREG uReg = ASMGetCR0();
4571	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4572	AssertRCReturn(rc, rc);
4573
4574	uReg = ASMGetCR3();
4575	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4576	AssertRCReturn(rc, rc);
4577
4578	uReg = ASMGetCR4();
4579	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4580	AssertRCReturn(rc, rc);
4581	return rc;
4582	}
4583
4584
4585	/**
4586	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4587	* the host-state area in the VMCS.
4588	*
4589	* @returns VBox status code.
4590	* @param pVCpu The cross context virtual CPU structure.
4591	*/
4592	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4593	{
4594	#if HC_ARCH_BITS == 64
4595	/**
4596	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4597	* requirements. See hmR0VmxExportHostSegmentRegs().
4598	*/
4599	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4600	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4601	{ \
4602	bool fValidSelector = true; \
4603	if ((selValue) & X86_SEL_LDT) \
4604	{ \
4605	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4606	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4607	} \
4608	if (fValidSelector) \
4609	{ \
4610	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4611	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4612	} \
4613	(selValue) = 0; \
4614	}
4615
4616	/*
4617	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4618	* will be messed up. We should -not- save the messed up state without restoring
4619	* the original host-state, see @bugref{7240}.
4620	*
4621	* This apparently can happen (most likely the FPU changes), deal with it rather than
4622	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4623	*/
4624	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4625	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4626	{
4627	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4628	pVCpu->idCpu));
4629	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4630	}
4631	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4632	#else
4633	RT_NOREF(pVCpu);
4634	#endif
4635
4636	/*
4637	* Host DS, ES, FS and GS segment registers.
4638	*/
4639	#if HC_ARCH_BITS == 64
4640	RTSEL uSelDS = ASMGetDS();
4641	RTSEL uSelES = ASMGetES();
4642	RTSEL uSelFS = ASMGetFS();
4643	RTSEL uSelGS = ASMGetGS();
4644	#else
4645	RTSEL uSelDS = 0;
4646	RTSEL uSelES = 0;
4647	RTSEL uSelFS = 0;
4648	RTSEL uSelGS = 0;
4649	#endif
4650
4651	/*
4652	* Host CS and SS segment registers.
4653	*/
4654	RTSEL uSelCS = ASMGetCS();
4655	RTSEL uSelSS = ASMGetSS();
4656
4657	/*
4658	* Host TR segment register.
4659	*/
4660	RTSEL uSelTR = ASMGetTR();
4661
4662	#if HC_ARCH_BITS == 64
4663	/*
4664	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4665	* gain VM-entry and restore them before we get preempted.
4666	*
4667	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4668	*/
4669	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4670	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4671	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4672	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4673	# undef VMXLOCAL_ADJUST_HOST_SEG
4674	#endif
4675
4676	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4677	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4678	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4679	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4680	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4681	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4682	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4683	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4684	Assert(uSelCS);
4685	Assert(uSelTR);
4686
4687	/* Write these host selector fields into the host-state area in the VMCS. */
4688	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4689	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4690	#if HC_ARCH_BITS == 64
4691	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4692	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4693	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4694	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4695	#else
4696	NOREF(uSelDS);
4697	NOREF(uSelES);
4698	NOREF(uSelFS);
4699	NOREF(uSelGS);
4700	#endif
4701	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4702	AssertRCReturn(rc, rc);
4703
4704	/*
4705	* Host GDTR and IDTR.
4706	*/
4707	RTGDTR Gdtr;
4708	RTIDTR Idtr;
4709	RT_ZERO(Gdtr);
4710	RT_ZERO(Idtr);
4711	ASMGetGDTR(&Gdtr);
4712	ASMGetIDTR(&Idtr);
4713	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4714	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4715	AssertRCReturn(rc, rc);
4716
4717	#if HC_ARCH_BITS == 64
4718	/*
4719	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4720	* them to the maximum limit (0xffff) on every VM-exit.
4721	*/
4722	if (Gdtr.cbGdt != 0xffff)
4723	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4724
4725	/*
4726	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4727	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4728	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4729	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4730	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4731	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4732	* at 0xffff on hosts where we are sure it won't cause trouble.
4733	*/
4734	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4735	if (Idtr.cbIdt < 0x0fff)
4736	# else
4737	if (Idtr.cbIdt != 0xffff)
4738	# endif
4739	{
4740	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4741	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4742	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4743	}
4744	#endif
4745
4746	/*
4747	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4748	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4749	* RPL should be too in most cases.
4750	*/
4751	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4752	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4753
4754	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4755	#if HC_ARCH_BITS == 64
4756	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4757
4758	/*
4759	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4760	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4761	* restoration if the host has something else. Task switching is not supported in 64-bit
4762	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4763	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4764	*
4765	* [1] See Intel spec. 3.5 "System Descriptor Types".
4766	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4767	*/
4768	PVM pVM = pVCpu->CTX_SUFF(pVM);
4769	Assert(pDesc->System.u4Type == 11);
4770	if ( pDesc->System.u16LimitLow != 0x67
4771	\|\| pDesc->System.u4LimitHigh)
4772	{
4773	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4774	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4775	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4776	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4777	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4778	}
4779
4780	/*
4781	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4782	*/
4783	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4784	{
4785	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4786	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4787	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4788	{
4789	/* The GDT is read-only but the writable GDT is available. */
4790	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4791	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4792	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4793	AssertRCReturn(rc, rc);
4794	}
4795	}
4796	#else
4797	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4798	#endif
4799	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4800	AssertRCReturn(rc, rc);
4801
4802	/*
4803	* Host FS base and GS base.
4804	*/
4805	#if HC_ARCH_BITS == 64
4806	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4807	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4808	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4809	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4810	AssertRCReturn(rc, rc);
4811
4812	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4813	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4814	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4815	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4816	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4817	#endif
4818	return VINF_SUCCESS;
4819	}
4820
4821
4822	/**
4823	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4824	* host-state area of the VMCS.
4825	*
4826	* These MSRs will be automatically restored on the host after every successful
4827	* VM-exit.
4828	*
4829	* @returns VBox status code.
4830	* @param pVCpu The cross context virtual CPU structure.
4831	*
4832	* @remarks No-long-jump zone!!!
4833	*/
4834	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4835	{
4836	AssertPtr(pVCpu);
4837
4838	/*
4839	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4840	* rather than swapping them on every VM-entry.
4841	*/
4842	hmR0VmxLazySaveHostMsrs(pVCpu);
4843
4844	/*
4845	* Host Sysenter MSRs.
4846	*/
4847	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4848	#if HC_ARCH_BITS == 32
4849	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4850	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4851	#else
4852	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4853	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4854	#endif
4855	AssertRCReturn(rc, rc);
4856
4857	/*
4858	* Host EFER MSR.
4859	*
4860	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4861	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4862	*/
4863	PVM pVM = pVCpu->CTX_SUFF(pVM);
4864	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4865	{
4866	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4867	AssertRCReturn(rc, rc);
4868	}
4869
4870	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4871	* hmR0VmxExportGuestEntryExitCtls(). */
4872
4873	return VINF_SUCCESS;
4874	}
4875
4876
4877	/**
4878	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4879	*
4880	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4881	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4882	*
4883	* @returns true if we need to load guest EFER, false otherwise.
4884	* @param pVCpu The cross context virtual CPU structure.
4885	*
4886	* @remarks Requires EFER, CR4.
4887	* @remarks No-long-jump zone!!!
4888	*/
4889	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4890	{
4891	#ifdef HMVMX_ALWAYS_SWAP_EFER
4892	RT_NOREF(pVCpu);
4893	return true;
4894	#else
4895	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4896	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4897	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4898	if (CPUMIsGuestInLongModeEx(pCtx))
4899	return false;
4900	#endif
4901
4902	PVM pVM = pVCpu->CTX_SUFF(pVM);
4903	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4904	uint64_t const u64GuestEfer = pCtx->msrEFER;
4905
4906	/*
4907	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4908	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4909	*/
4910	if ( CPUMIsGuestInLongModeEx(pCtx)
4911	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4912	return true;
4913
4914	/*
4915	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4916	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4917	*
4918	* See Intel spec. 4.5 "IA-32e Paging".
4919	* See Intel spec. 4.1.1 "Three Paging Modes".
4920	*
4921	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4922	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4923	*/
4924	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4925	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4926	if ( (pCtx->cr4 & X86_CR4_PAE)
4927	&& (pCtx->cr0 & X86_CR0_PG)
4928	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4929	{
4930	/* Assert that host is NX capable. */
4931	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4932	return true;
4933	}
4934
4935	return false;
4936	#endif
4937	}
4938
4939	/**
4940	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4941	* VMCS.
4942	*
4943	* This is typically required when the guest changes paging mode.
4944	*
4945	* @returns VBox status code.
4946	* @param pVCpu The cross context virtual CPU structure.
4947	* @param pVmxTransient The VMX-transient structure.
4948	*
4949	* @remarks Requires EFER.
4950	* @remarks No-long-jump zone!!!
4951	*/
4952	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4953	{
4954	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4955	{
4956	PVM pVM = pVCpu->CTX_SUFF(pVM);
4957	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4958
4959	/*
4960	* VM-entry controls.
4961	*/
4962	{
4963	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4964	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4965
4966	/*
4967	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4968	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4969	*
4970	* For nested-guests, this is a mandatory VM-entry control. It's also
4971	* required because we do not want to leak host bits to the nested-guest.
4972	*/
4973	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4974
4975	/*
4976	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4977	*
4978	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4979	* required to get the nested-guest working with hardware-assisted VMX execution.
4980	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4981	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4982	* here rather than while merging the guest VMCS controls.
4983	*/
4984	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4985	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4986	else
4987	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4988
4989	/*
4990	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4991	*
4992	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4993	* regardless of whether the nested-guest VMCS specifies it because we are free to
4994	* load whatever MSRs we require and we do not need to modify the guest visible copy
4995	* of the VM-entry MSR load area.
4996	*/
4997	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4998	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4999	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
5000	else
5001	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
5002
5003	/*
5004	* The following should -not- be set (since we're not in SMM mode):
5005	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
5006	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
5007	*/
5008
5009	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
5010	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
5011
5012	if ((fVal & fZap) == fVal)
5013	{ /* likely */ }
5014	else
5015	{
5016	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
5017	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
5018	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
5019	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5020	}
5021
5022	/* Commit it to the VMCS. */
5023	if (pVmcsInfo->u32EntryCtls != fVal)
5024	{
5025	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
5026	AssertRCReturn(rc, rc);
5027	pVmcsInfo->u32EntryCtls = fVal;
5028	}
5029	}
5030
5031	/*
5032	* VM-exit controls.
5033	*/
5034	{
5035	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
5036	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
5037
5038	/*
5039	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
5040	* supported the 1-setting of this bit.
5041	*
5042	* For nested-guests, we set the "save debug controls" as the converse
5043	* "load debug controls" is mandatory for nested-guests anyway.
5044	*/
5045	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
5046
5047	/*
5048	* Set the host long mode active (EFER.LMA) bit (which Intel calls
5049	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
5050	* host EFER.LMA and EFER.LME bit to this value. See assertion in
5051	* hmR0VmxExportHostMsrs().
5052	*
5053	* For nested-guests, we always set this bit as we do not support 32-bit
5054	* hosts.
5055	*/
5056	#if HC_ARCH_BITS == 64
5057	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5058	#else
5059	Assert(!pVmxTransient->fIsNestedGuest);
5060	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
5061	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
5062	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
5063	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
5064	{
5065	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
5066	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
5067	}
5068	else
5069	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
5070	#endif
5071
5072	/*
5073	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
5074	*
5075	* For nested-guests, we should use the "save IA32_EFER" control if we also
5076	* used the "load IA32_EFER" control while exporting VM-entry controls.
5077	*/
5078	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
5079	&& hmR0VmxShouldSwapEferMsr(pVCpu))
5080	{
5081	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
5082	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
5083	}
5084
5085	/*
5086	* Enable saving of the VMX-preemption timer value on VM-exit.
5087	* For nested-guests, currently not exposed/used.
5088	*/
5089	if ( pVM->hm.s.vmx.fUsePreemptTimer
5090	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
5091	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
5092
5093	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
5094	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
5095
5096	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
5097	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
5098	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
5099
5100	if ((fVal & fZap) == fVal)
5101	{ /* likely */ }
5102	else
5103	{
5104	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
5105	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
5106	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
5107	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
5108	}
5109
5110	/* Commit it to the VMCS. */
5111	if (pVmcsInfo->u32ExitCtls != fVal)
5112	{
5113	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
5114	AssertRCReturn(rc, rc);
5115	pVmcsInfo->u32ExitCtls = fVal;
5116	}
5117	}
5118
5119	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
5120	}
5121	return VINF_SUCCESS;
5122	}
5123
5124
5125	/**
5126	* Sets the TPR threshold in the VMCS.
5127	*
5128	* @returns VBox status code.
5129	* @param pVCpu The cross context virtual CPU structure.
5130	* @param pVmcsInfo The VMCS info. object.
5131	* @param u32TprThreshold The TPR threshold (task-priority class only).
5132	*/
5133	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
5134	{
5135	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
5136	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
5137	RT_NOREF2(pVCpu, pVmcsInfo);
5138	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
5139	}
5140
5141
5142	/**
5143	* Exports the guest APIC TPR state into the VMCS.
5144	*
5145	* @returns VBox status code.
5146	* @param pVCpu The cross context virtual CPU structure.
5147	* @param pVmxTransient The VMX-transient structure.
5148	*
5149	* @remarks No-long-jump zone!!!
5150	*/
5151	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5152	{
5153	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
5154	{
5155	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5156
5157	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5158	if (!pVmxTransient->fIsNestedGuest)
5159	{
5160	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
5161	&& APICIsEnabled(pVCpu))
5162	{
5163	/*
5164	* Setup TPR shadowing.
5165	*/
5166	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
5167	{
5168	bool fPendingIntr = false;
5169	uint8_t u8Tpr = 0;
5170	uint8_t u8PendingIntr = 0;
5171	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
5172	AssertRCReturn(rc, rc);
5173
5174	/*
5175	* If there are interrupts pending but masked by the TPR, instruct VT-x to
5176	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
5177	* priority of the pending interrupt so we can deliver the interrupt. If there
5178	* are no interrupts pending, set threshold to 0 to not cause any
5179	* TPR-below-threshold VM-exits.
5180	*/
5181	Assert(pVmcsInfo->pbVirtApic);
5182	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
5183	uint32_t u32TprThreshold = 0;
5184	if (fPendingIntr)
5185	{
5186	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
5187	(which is the Task-Priority Class). */
5188	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
5189	const uint8_t u8TprPriority = u8Tpr >> 4;
5190	if (u8PendingPriority <= u8TprPriority)
5191	u32TprThreshold = u8PendingPriority;
5192	}
5193
5194	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
5195	AssertRCReturn(rc, rc);
5196	}
5197	}
5198	}
5199	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
5200	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
5201	}
5202	return VINF_SUCCESS;
5203	}
5204
5205
5206	/**
5207	* Gets the guest interruptibility-state.
5208	*
5209	* @returns Guest's interruptibility-state.
5210	* @param pVCpu The cross context virtual CPU structure.
5211	* @param pVmxTransient The VMX-transient structure.
5212	*
5213	* @remarks No-long-jump zone!!!
5214	*/
5215	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5216	{
5217	/*
5218	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
5219	*/
5220	uint32_t fIntrState = 0;
5221	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5222	{
5223	/* If inhibition is active, RIP and RFLAGS should've been updated
5224	(i.e. read previously from the VMCS or from ring-3). */
5225	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5226	#ifdef VBOX_STRICT
5227	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
5228	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5229	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
5230	#endif
5231	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5232	{
5233	if (pCtx->eflags.Bits.u1IF)
5234	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5235	else
5236	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5237	}
5238	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5239	{
5240	/*
5241	* We can clear the inhibit force flag as even if we go back to the recompiler
5242	* without executing guest code in VT-x, the flag's condition to be cleared is
5243	* met and thus the cleared state is correct.
5244	*/
5245	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5246	}
5247	}
5248
5249	/*
5250	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5251	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5252	* setting this would block host-NMIs and IRET will not clear the blocking.
5253	*
5254	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5255	*
5256	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5257	*/
5258	if ( hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
5259	&& CPUMIsGuestNmiBlocking(pVCpu))
5260	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5261
5262	return fIntrState;
5263	}
5264
5265
5266	/**
5267	* Exports the exception intercepts required for guest execution in the VMCS.
5268	*
5269	* @returns VBox status code.
5270	* @param pVCpu The cross context virtual CPU structure.
5271	* @param pVmxTransient The VMX-transient structure.
5272	*
5273	* @remarks No-long-jump zone!!!
5274	*/
5275	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5276	{
5277	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5278	{
5279	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5280	if ( !pVmxTransient->fIsNestedGuest
5281	&& pVCpu->hm.s.fGIMTrapXcptUD)
5282	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5283	else
5284	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5285
5286	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5287	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5288	}
5289	return VINF_SUCCESS;
5290	}
5291
5292
5293	/**
5294	* Exports the guest's RIP into the guest-state area in the VMCS.
5295	*
5296	* @returns VBox status code.
5297	* @param pVCpu The cross context virtual CPU structure.
5298	*
5299	* @remarks No-long-jump zone!!!
5300	*/
5301	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5302	{
5303	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5304	{
5305	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5306
5307	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5308	AssertRCReturn(rc, rc);
5309
5310	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5311	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5312	}
5313	return VINF_SUCCESS;
5314	}
5315
5316
5317	/**
5318	* Exports the guest's RSP into the guest-state area in the VMCS.
5319	*
5320	* @returns VBox status code.
5321	* @param pVCpu The cross context virtual CPU structure.
5322	*
5323	* @remarks No-long-jump zone!!!
5324	*/
5325	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5326	{
5327	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5328	{
5329	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5330
5331	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5332	AssertRCReturn(rc, rc);
5333
5334	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5335	}
5336	return VINF_SUCCESS;
5337	}
5338
5339
5340	/**
5341	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5342	*
5343	* @returns VBox status code.
5344	* @param pVCpu The cross context virtual CPU structure.
5345	* @param pVmxTransient The VMX-transient structure.
5346	*
5347	* @remarks No-long-jump zone!!!
5348	*/
5349	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5350	{
5351	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5352	{
5353	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5354
5355	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5356	Let us assert it as such and use 32-bit VMWRITE. */
5357	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5358	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5359	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5360	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5361
5362	/*
5363	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5364	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5365	* can run the real-mode guest code under Virtual 8086 mode.
5366	*/
5367	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5368	if (pVmcsInfo->RealMode.fRealOnV86Active)
5369	{
5370	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5371	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5372	Assert(!pVmxTransient->fIsNestedGuest);
5373	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5374	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5375	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5376	}
5377
5378	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5379	AssertRCReturn(rc, rc);
5380
5381	/*
5382	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5383	*
5384	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5385	* through the hypervisor debugger using EFLAGS.TF.
5386	*/
5387	if ( !pVmxTransient->fIsNestedGuest
5388	&& !pVCpu->hm.s.fSingleInstruction
5389	&& fEFlags.Bits.u1TF)
5390	{
5391	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5392	* premature trips to ring-3 esp since IEM does not yet handle it. */
5393	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5394	AssertRCReturn(rc, rc);
5395	}
5396	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5397	* nested-guest VMCS. */
5398
5399	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5400	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5401	}
5402	return VINF_SUCCESS;
5403	}
5404
5405
5406	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5407	/**
5408	* Copies the nested-guest VMCS to the shadow VMCS.
5409	*
5410	* @returns VBox status code.
5411	* @param pVCpu The cross context virtual CPU structure.
5412	* @param pVmcsInfo The VMCS info. object.
5413	*
5414	* @remarks No-long-jump zone!!!
5415	*/
5416	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5417	{
5418	PVM pVM = pVCpu->CTX_SUFF(pVM);
5419	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5420
5421	/*
5422	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5423	* current VMCS, as we may try saving guest lazy MSRs.
5424	*
5425	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5426	* calling the import VMCS code which is currently performing the guest MSR reads
5427	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5428	* and the rest of the VMX leave session machinery.
5429	*/
5430	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5431
5432	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5433	if (RT_SUCCESS(rc))
5434	{
5435	/*
5436	* Copy all guest read/write VMCS fields.
5437	*
5438	* We don't check for VMWRITE failures here for performance reasons and
5439	* because they are not expected to fail, barring irrecoverable conditions
5440	* like hardware errors.
5441	*/
5442	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5443	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5444	{
5445	uint64_t u64Val;
5446	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5447	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5448	VMXWriteVmcs64(uVmcsField, u64Val);
5449	}
5450
5451	/*
5452	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5453	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5454	*/
5455	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5456	{
5457	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5458	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5459	{
5460	uint64_t u64Val;
5461	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5462	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5463	VMXWriteVmcs64(uVmcsField, u64Val);
5464	}
5465	}
5466
5467	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5468	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5469	}
5470
5471	ASMSetFlags(fEFlags);
5472	return rc;
5473	}
5474
5475
5476	/**
5477	* Copies the shadow VMCS to the nested-guest VMCS.
5478	*
5479	* @returns VBox status code.
5480	* @param pVCpu The cross context virtual CPU structure.
5481	* @param pVmcsInfo The VMCS info. object.
5482	*
5483	* @remarks Called with interrupts disabled.
5484	*/
5485	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5486	{
5487	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5488	PVM pVM = pVCpu->CTX_SUFF(pVM);
5489	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5490
5491	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5492	if (RT_SUCCESS(rc))
5493	{
5494	/*
5495	* Copy guest read/write fields from the shadow VMCS.
5496	* Guest read-only fields cannot be modified, so no need to copy them.
5497	*
5498	* We don't check for VMREAD failures here for performance reasons and
5499	* because they are not expected to fail, barring irrecoverable conditions
5500	* like hardware errors.
5501	*/
5502	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5503	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5504	{
5505	uint64_t u64Val;
5506	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5507	VMXReadVmcs64(uVmcsField, &u64Val);
5508	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5509	}
5510
5511	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5512	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5513	}
5514	return rc;
5515	}
5516
5517
5518	/**
5519	* Enables VMCS shadowing for the given VMCS info. object.
5520	*
5521	* @param pVCpu The cross context virtual CPU structure.
5522	* @param pVmcsInfo The VMCS info. object.
5523	*
5524	* @remarks No-long-jump zone!!!
5525	*/
5526	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5527	{
5528	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5529
5530	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5531	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5532	{
5533	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5534	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5535	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5536	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5537	AssertRC(rc);
5538
5539	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5540	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5541	Log4Func(("Enabled\n"));
5542	}
5543	}
5544
5545
5546	/**
5547	* Disables VMCS shadowing for the given VMCS info. object.
5548	*
5549	* @param pVCpu The cross context virtual CPU structure.
5550	* @param pVmcsInfo The VMCS info. object.
5551	*
5552	* @remarks No-long-jump zone!!!
5553	*/
5554	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5555	{
5556	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5557
5558	/*
5559	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5560	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5561	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5562	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5563	*
5564	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5565	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5566	*/
5567	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5568	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5569	{
5570	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5571	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5572	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5573	AssertRC(rc);
5574
5575	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5576	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5577	Log4Func(("Disabled\n"));
5578	}
5579	}
5580	#endif
5581
5582
5583	/**
5584	* Exports the guest hardware-virtualization state.
5585	*
5586	* @returns VBox status code.
5587	* @param pVCpu The cross context virtual CPU structure.
5588	* @param pVmxTransient The VMX-transient structure.
5589	*
5590	* @remarks No-long-jump zone!!!
5591	*/
5592	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5593	{
5594	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5595	{
5596	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5597	/*
5598	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5599	* VMCS shadowing.
5600	*/
5601	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5602	{
5603	/*
5604	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5605	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5606	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5607	*
5608	* We check for VMX root mode here in case the guest executes VMXOFF without
5609	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5610	* not clear the current VMCS pointer.
5611	*/
5612	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5613	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5614	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5615	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5616	{
5617	/* Paranoia. */
5618	Assert(!pVmxTransient->fIsNestedGuest);
5619
5620	/*
5621	* For performance reasons, also check if the guest hypervisor's current VMCS
5622	* was newly loaded or modified before copying it to the shadow VMCS.
5623	*/
5624	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5625	{
5626	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5627	AssertRCReturn(rc, rc);
5628	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5629	}
5630	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5631	}
5632	else
5633	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5634	}
5635	#else
5636	NOREF(pVmxTransient);
5637	#endif
5638	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5639	}
5640	return VINF_SUCCESS;
5641	}
5642
5643
5644	/**
5645	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5646	*
5647	* The guest FPU state is always pre-loaded hence we don't need to bother about
5648	* sharing FPU related CR0 bits between the guest and host.
5649	*
5650	* @returns VBox status code.
5651	* @param pVCpu The cross context virtual CPU structure.
5652	* @param pVmxTransient The VMX-transient structure.
5653	*
5654	* @remarks No-long-jump zone!!!
5655	*/
5656	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5657	{
5658	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5659	{
5660	PVM pVM = pVCpu->CTX_SUFF(pVM);
5661	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5662
5663	/*
5664	* Figure out fixed CR0 bits in VMX operation.
5665	*/
5666	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5667	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5668	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5669	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5670	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5671	else
5672	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5673
5674	if (!pVmxTransient->fIsNestedGuest)
5675	{
5676	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5677	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5678	uint64_t const u64ShadowCr0 = u64GuestCr0;
5679	Assert(!RT_HI_U32(u64GuestCr0));
5680
5681	/*
5682	* Setup VT-x's view of the guest CR0.
5683	*/
5684	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5685	if (pVM->hm.s.fNestedPaging)
5686	{
5687	if (CPUMIsGuestPagingEnabled(pVCpu))
5688	{
5689	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5690	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5691	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5692	}
5693	else
5694	{
5695	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5696	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5697	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5698	}
5699
5700	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5701	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5702	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5703	}
5704	else
5705	{
5706	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5707	u64GuestCr0 \|= X86_CR0_WP;
5708	}
5709
5710	/*
5711	* Guest FPU bits.
5712	*
5713	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5714	* using CR0.TS.
5715	*
5716	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5717	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5718	*/
5719	u64GuestCr0 \|= X86_CR0_NE;
5720
5721	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5722	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5723
5724	/*
5725	* Update exception intercepts.
5726	*/
5727	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5728	if (pVmcsInfo->RealMode.fRealOnV86Active)
5729	{
5730	Assert(PDMVmmDevHeapIsEnabled(pVM));
5731	Assert(pVM->hm.s.vmx.pRealModeTSS);
5732	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5733	}
5734	else
5735	{
5736	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5737	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5738	if (fInterceptMF)
5739	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5740	}
5741
5742	/* Additional intercepts for debugging, define these yourself explicitly. */
5743	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5744	uXcptBitmap \|= 0
5745	\| RT_BIT(X86_XCPT_BP)
5746	\| RT_BIT(X86_XCPT_DE)
5747	\| RT_BIT(X86_XCPT_NM)
5748	\| RT_BIT(X86_XCPT_TS)
5749	\| RT_BIT(X86_XCPT_UD)
5750	\| RT_BIT(X86_XCPT_NP)
5751	\| RT_BIT(X86_XCPT_SS)
5752	\| RT_BIT(X86_XCPT_GP)
5753	\| RT_BIT(X86_XCPT_PF)
5754	\| RT_BIT(X86_XCPT_MF)
5755	;
5756	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5757	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5758	#endif
5759	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5760	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5761	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5762
5763	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5764	u64GuestCr0 \|= fSetCr0;
5765	u64GuestCr0 &= fZapCr0;
5766	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5767
5768	/* Commit the CR0 and related fields to the guest VMCS. */
5769	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5770	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5771	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5772	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5773	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5774	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5775	AssertRCReturn(rc, rc);
5776
5777	/* Update our caches. */
5778	pVmcsInfo->u32ProcCtls = uProcCtls;
5779	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5780
5781	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5782	}
5783	else
5784	{
5785	/*
5786	* With nested-guests, we may have extended the guest/host mask here since we
5787	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5788	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5789	* originally supplied. We must copy those bits from the nested-guest CR0 into
5790	* the nested-guest CR0 read-shadow.
5791	*/
5792	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5793	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5794	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5795	Assert(!RT_HI_U32(u64GuestCr0));
5796	Assert(u64GuestCr0 & X86_CR0_NE);
5797
5798	/*
5799	* Apply the hardware specified fixed CR0 bits and enable caching.
5800	* Note! We could be altering our VMX emulation's fixed bits. We thus
5801	* need to re-apply them while importing CR0.
5802	*/
5803	u64GuestCr0 \|= fSetCr0;
5804	u64GuestCr0 &= fZapCr0;
5805	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5806
5807	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5808	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5809	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5810	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5811	AssertRCReturn(rc, rc);
5812
5813	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5814	}
5815
5816	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5817	}
5818
5819	return VINF_SUCCESS;
5820	}
5821
5822
5823	/**
5824	* Exports the guest control registers (CR3, CR4) into the guest-state area
5825	* in the VMCS.
5826	*
5827	* @returns VBox strict status code.
5828	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5829	* without unrestricted guest access and the VMMDev is not presently
5830	* mapped (e.g. EFI32).
5831	*
5832	* @param pVCpu The cross context virtual CPU structure.
5833	* @param pVmxTransient The VMX-transient structure.
5834	*
5835	* @remarks No-long-jump zone!!!
5836	*/
5837	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5838	{
5839	int rc = VINF_SUCCESS;
5840	PVM pVM = pVCpu->CTX_SUFF(pVM);
5841
5842	/*
5843	* Guest CR2.
5844	* It's always loaded in the assembler code. Nothing to do here.
5845	*/
5846
5847	/*
5848	* Guest CR3.
5849	*/
5850	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5851	{
5852	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5853
5854	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5855	if (pVM->hm.s.fNestedPaging)
5856	{
5857	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5858	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5859
5860	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5861	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5862	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5863	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5864
5865	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5866	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5867	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5868
5869	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5870	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5871	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5872	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5873	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5874	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5875	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5876
5877	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5878	AssertRCReturn(rc, rc);
5879
5880	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5881	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5882	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5883	{
5884	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5885	if (CPUMIsGuestInPAEModeEx(pCtx))
5886	{
5887	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5888	AssertRCReturn(rc, rc);
5889	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5890	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5891	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5892	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5893	AssertRCReturn(rc, rc);
5894	}
5895
5896	/*
5897	* The guest's view of its CR3 is unblemished with nested paging when the
5898	* guest is using paging or we have unrestricted guest execution to handle
5899	* the guest when it's not using paging.
5900	*/
5901	GCPhysGuestCr3 = pCtx->cr3;
5902	}
5903	else
5904	{
5905	/*
5906	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5907	* thinks it accesses physical memory directly, we use our identity-mapped
5908	* page table to map guest-linear to guest-physical addresses. EPT takes care
5909	* of translating it to host-physical addresses.
5910	*/
5911	RTGCPHYS GCPhys;
5912	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5913
5914	/* We obtain it here every time as the guest could have relocated this PCI region. */
5915	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5916	if (RT_SUCCESS(rc))
5917	{ /* likely */ }
5918	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5919	{
5920	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5921	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5922	}
5923	else
5924	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5925
5926	GCPhysGuestCr3 = GCPhys;
5927	}
5928
5929	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5930	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5931	AssertRCReturn(rc, rc);
5932	}
5933	else
5934	{
5935	/* Non-nested paging case, just use the hypervisor's CR3. */
5936	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5937
5938	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5939	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5940	AssertRCReturn(rc, rc);
5941	}
5942
5943	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5944	}
5945
5946	/*
5947	* Guest CR4.
5948	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5949	*/
5950	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5951	{
5952	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5953	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5954
5955	/*
5956	* Figure out fixed CR4 bits in VMX operation.
5957	*/
5958	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5959	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5960	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5961
5962	/*
5963	* With nested-guests, we may have extended the guest/host mask here (since we
5964	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5965	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5966	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5967	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5968	*/
5969	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5970	uint64_t u64GuestCr4 = pCtx->cr4;
5971	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5972	? pCtx->cr4
5973	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5974	Assert(!RT_HI_U32(u64GuestCr4));
5975
5976	/*
5977	* Setup VT-x's view of the guest CR4.
5978	*
5979	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5980	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5981	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5982	*
5983	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5984	*/
5985	if (pVmcsInfo->RealMode.fRealOnV86Active)
5986	{
5987	Assert(pVM->hm.s.vmx.pRealModeTSS);
5988	Assert(PDMVmmDevHeapIsEnabled(pVM));
5989	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5990	}
5991
5992	if (pVM->hm.s.fNestedPaging)
5993	{
5994	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5995	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5996	{
5997	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5998	u64GuestCr4 \|= X86_CR4_PSE;
5999	/* Our identity mapping is a 32-bit page directory. */
6000	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
6001	}
6002	/* else use guest CR4.*/
6003	}
6004	else
6005	{
6006	Assert(!pVmxTransient->fIsNestedGuest);
6007
6008	/*
6009	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
6010	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
6011	*/
6012	switch (pVCpu->hm.s.enmShadowMode)
6013	{
6014	case PGMMODE_REAL: /* Real-mode. */
6015	case PGMMODE_PROTECTED: /* Protected mode without paging. */
6016	case PGMMODE_32_BIT: /* 32-bit paging. */
6017	{
6018	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
6019	break;
6020	}
6021
6022	case PGMMODE_PAE: /* PAE paging. */
6023	case PGMMODE_PAE_NX: /* PAE paging with NX. */
6024	{
6025	u64GuestCr4 \|= X86_CR4_PAE;
6026	break;
6027	}
6028
6029	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
6030	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
6031	#ifdef VBOX_ENABLE_64_BITS_GUESTS
6032	break;
6033	#endif
6034	default:
6035	AssertFailed();
6036	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6037	}
6038	}
6039
6040	/*
6041	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
6042	* Note! For nested-guests, we could be altering our VMX emulation's
6043	* fixed bits. We thus need to re-apply them while importing CR4.
6044	*/
6045	u64GuestCr4 \|= fSetCr4;
6046	u64GuestCr4 &= fZapCr4;
6047
6048	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
6049	/** @todo Fix to 64-bit when we drop 32-bit. */
6050	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
6051	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
6052	AssertRCReturn(rc, rc);
6053
6054	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
6055	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6056
6057	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
6058
6059	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
6060	}
6061	return rc;
6062	}
6063
6064
6065	/**
6066	* Exports the guest debug registers into the guest-state area in the VMCS.
6067	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
6068	*
6069	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
6070	*
6071	* @returns VBox status code.
6072	* @param pVCpu The cross context virtual CPU structure.
6073	* @param pVmxTransient The VMX-transient structure.
6074	*
6075	* @remarks No-long-jump zone!!!
6076	*/
6077	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6078	{
6079	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6080
6081	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
6082	* stepping. */
6083	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6084	if (pVmxTransient->fIsNestedGuest)
6085	{
6086	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
6087	AssertRCReturn(rc, rc);
6088	return VINF_SUCCESS;
6089	}
6090
6091	#ifdef VBOX_STRICT
6092	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
6093	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
6094	{
6095	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
6096	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
6097	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
6098	}
6099	#endif
6100
6101	bool fSteppingDB = false;
6102	bool fInterceptMovDRx = false;
6103	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6104	if (pVCpu->hm.s.fSingleInstruction)
6105	{
6106	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
6107	PVM pVM = pVCpu->CTX_SUFF(pVM);
6108	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
6109	{
6110	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
6111	Assert(fSteppingDB == false);
6112	}
6113	else
6114	{
6115	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
6116	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
6117	pVCpu->hm.s.fClearTrapFlag = true;
6118	fSteppingDB = true;
6119	}
6120	}
6121
6122	uint32_t u32GuestDr7;
6123	if ( fSteppingDB
6124	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
6125	{
6126	/*
6127	* Use the combined guest and host DRx values found in the hypervisor register set
6128	* because the hypervisor debugger has breakpoints active or someone is single stepping
6129	* on the host side without a monitor trap flag.
6130	*
6131	* Note! DBGF expects a clean DR6 state before executing guest code.
6132	*/
6133	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6134	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6135	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
6136	{
6137	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6138	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
6139	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
6140	}
6141	else
6142	#endif
6143	if (!CPUMIsHyperDebugStateActive(pVCpu))
6144	{
6145	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
6146	Assert(CPUMIsHyperDebugStateActive(pVCpu));
6147	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6148	}
6149
6150	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
6151	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
6152	pVCpu->hm.s.fUsingHyperDR7 = true;
6153	fInterceptMovDRx = true;
6154	}
6155	else
6156	{
6157	/*
6158	* If the guest has enabled debug registers, we need to load them prior to
6159	* executing guest code so they'll trigger at the right time.
6160	*/
6161	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
6162	{
6163	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6164	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6165	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
6166	{
6167	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6168	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
6169	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
6170	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6171	}
6172	else
6173	#endif
6174	if (!CPUMIsGuestDebugStateActive(pVCpu))
6175	{
6176	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
6177	Assert(CPUMIsGuestDebugStateActive(pVCpu));
6178	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6179	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
6180	}
6181	Assert(!fInterceptMovDRx);
6182	}
6183	/*
6184	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
6185	* must intercept #DB in order to maintain a correct DR6 guest value, and
6186	* because we need to intercept it to prevent nested #DBs from hanging the
6187	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
6188	*/
6189	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
6190	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
6191	&& !CPUMIsGuestDebugStateActive(pVCpu))
6192	#else
6193	else if (!CPUMIsGuestDebugStateActive(pVCpu))
6194	#endif
6195	{
6196	fInterceptMovDRx = true;
6197	}
6198
6199	/* Update DR7 with the actual guest value. */
6200	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
6201	pVCpu->hm.s.fUsingHyperDR7 = false;
6202	}
6203
6204	if (fInterceptMovDRx)
6205	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
6206	else
6207	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
6208
6209	/*
6210	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
6211	* monitor-trap flag and update our cache.
6212	*/
6213	if (uProcCtls != pVmcsInfo->u32ProcCtls)
6214	{
6215	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6216	AssertRCReturn(rc2, rc2);
6217	pVmcsInfo->u32ProcCtls = uProcCtls;
6218	}
6219
6220	/*
6221	* Update guest DR7.
6222	*/
6223	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6224	AssertRCReturn(rc, rc);
6225
6226	/*
6227	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6228	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6229	*
6230	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6231	*/
6232	if (fSteppingDB)
6233	{
6234	Assert(pVCpu->hm.s.fSingleInstruction);
6235	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6236
6237	uint32_t fIntrState = 0;
6238	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6239	AssertRCReturn(rc, rc);
6240
6241	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6242	{
6243	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6244	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6245	AssertRCReturn(rc, rc);
6246	}
6247	}
6248
6249	return VINF_SUCCESS;
6250	}
6251
6252
6253	#ifdef VBOX_STRICT
6254	/**
6255	* Strict function to validate segment registers.
6256	*
6257	* @param pVCpu The cross context virtual CPU structure.
6258	* @param pVmcsInfo The VMCS info. object.
6259	*
6260	* @remarks Will import guest CR0 on strict builds during validation of
6261	* segments.
6262	*/
6263	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6264	{
6265	/*
6266	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6267	*
6268	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6269	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6270	* unusable bit and doesn't change the guest-context value.
6271	*/
6272	PVM pVM = pVCpu->CTX_SUFF(pVM);
6273	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6274	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6275	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6276	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6277	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6278	{
6279	/* Protected mode checks */
6280	/* CS */
6281	Assert(pCtx->cs.Attr.n.u1Present);
6282	Assert(!(pCtx->cs.Attr.u & 0xf00));
6283	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6284	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6285	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6286	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6287	\|\| (pCtx->cs.Attr.n.u1Granularity));
6288	/* CS cannot be loaded with NULL in protected mode. */
6289	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6290	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6291	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6292	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6293	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6294	else
6295	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6296	/* SS */
6297	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6298	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6299	if ( !(pCtx->cr0 & X86_CR0_PE)
6300	\|\| pCtx->cs.Attr.n.u4Type == 3)
6301	{
6302	Assert(!pCtx->ss.Attr.n.u2Dpl);
6303	}
6304	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6305	{
6306	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6307	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6308	Assert(pCtx->ss.Attr.n.u1Present);
6309	Assert(!(pCtx->ss.Attr.u & 0xf00));
6310	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6311	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6312	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6313	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6314	\|\| (pCtx->ss.Attr.n.u1Granularity));
6315	}
6316	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6317	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6318	{
6319	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6320	Assert(pCtx->ds.Attr.n.u1Present);
6321	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6322	Assert(!(pCtx->ds.Attr.u & 0xf00));
6323	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6324	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6325	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6326	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6327	\|\| (pCtx->ds.Attr.n.u1Granularity));
6328	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6329	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6330	}
6331	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6332	{
6333	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6334	Assert(pCtx->es.Attr.n.u1Present);
6335	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6336	Assert(!(pCtx->es.Attr.u & 0xf00));
6337	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6338	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6339	\|\| !(pCtx->es.Attr.n.u1Granularity));
6340	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6341	\|\| (pCtx->es.Attr.n.u1Granularity));
6342	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6343	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6344	}
6345	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6346	{
6347	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6348	Assert(pCtx->fs.Attr.n.u1Present);
6349	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6350	Assert(!(pCtx->fs.Attr.u & 0xf00));
6351	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6352	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6353	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6354	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6355	\|\| (pCtx->fs.Attr.n.u1Granularity));
6356	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6357	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6358	}
6359	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6360	{
6361	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6362	Assert(pCtx->gs.Attr.n.u1Present);
6363	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6364	Assert(!(pCtx->gs.Attr.u & 0xf00));
6365	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6366	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6367	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6368	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6369	\|\| (pCtx->gs.Attr.n.u1Granularity));
6370	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6371	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6372	}
6373	/* 64-bit capable CPUs. */
6374	# if HC_ARCH_BITS == 64
6375	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6376	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6377	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6378	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6379	# endif
6380	}
6381	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6382	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6383	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6384	{
6385	/* Real and v86 mode checks. */
6386	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6387	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6388	if (pVmcsInfo->RealMode.fRealOnV86Active)
6389	{
6390	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6391	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6392	}
6393	else
6394	{
6395	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6396	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6397	}
6398
6399	/* CS */
6400	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6401	Assert(pCtx->cs.u32Limit == 0xffff);
6402	Assert(u32CSAttr == 0xf3);
6403	/* SS */
6404	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6405	Assert(pCtx->ss.u32Limit == 0xffff);
6406	Assert(u32SSAttr == 0xf3);
6407	/* DS */
6408	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6409	Assert(pCtx->ds.u32Limit == 0xffff);
6410	Assert(u32DSAttr == 0xf3);
6411	/* ES */
6412	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6413	Assert(pCtx->es.u32Limit == 0xffff);
6414	Assert(u32ESAttr == 0xf3);
6415	/* FS */
6416	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6417	Assert(pCtx->fs.u32Limit == 0xffff);
6418	Assert(u32FSAttr == 0xf3);
6419	/* GS */
6420	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6421	Assert(pCtx->gs.u32Limit == 0xffff);
6422	Assert(u32GSAttr == 0xf3);
6423	/* 64-bit capable CPUs. */
6424	# if HC_ARCH_BITS == 64
6425	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6426	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6427	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6428	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6429	# endif
6430	}
6431	}
6432	#endif /* VBOX_STRICT */
6433
6434
6435	/**
6436	* Exports a guest segment register into the guest-state area in the VMCS.
6437	*
6438	* @returns VBox status code.
6439	* @param pVCpu The cross context virtual CPU structure.
6440	* @param pVmcsInfo The VMCS info. object.
6441	* @param iSegReg The segment register number (X86_SREG_XXX).
6442	* @param pSelReg Pointer to the segment selector.
6443	*
6444	* @remarks No-long-jump zone!!!
6445	*/
6446	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6447	{
6448	Assert(iSegReg < X86_SREG_COUNT);
6449	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6450	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6451	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6452	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6453
6454	uint32_t u32Access = pSelReg->Attr.u;
6455	if (pVmcsInfo->RealMode.fRealOnV86Active)
6456	{
6457	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6458	u32Access = 0xf3;
6459	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6460	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6461	RT_NOREF_PV(pVCpu);
6462	}
6463	else
6464	{
6465	/*
6466	* The way to differentiate between whether this is really a null selector or was just
6467	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6468	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6469	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6470	* NULL selectors loaded in protected-mode have their attribute as 0.
6471	*/
6472	if (!u32Access)
6473	u32Access = X86DESCATTR_UNUSABLE;
6474	}
6475
6476	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6477	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6478	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6479
6480	/*
6481	* Commit it to the VMCS.
6482	*/
6483	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6484	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6485	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6486	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6487	AssertRCReturn(rc, rc);
6488	return rc;
6489	}
6490
6491
6492	/**
6493	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6494	* area in the VMCS.
6495	*
6496	* @returns VBox status code.
6497	* @param pVCpu The cross context virtual CPU structure.
6498	* @param pVmxTransient The VMX-transient structure.
6499	*
6500	* @remarks Will import guest CR0 on strict builds during validation of
6501	* segments.
6502	* @remarks No-long-jump zone!!!
6503	*/
6504	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6505	{
6506	int rc = VERR_INTERNAL_ERROR_5;
6507	PVM pVM = pVCpu->CTX_SUFF(pVM);
6508	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6509	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6510
6511	/*
6512	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6513	*/
6514	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6515	{
6516	#ifdef VBOX_WITH_REM
6517	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6518	{
6519	Assert(!pVmxTransient->fIsNestedGuest);
6520	Assert(pVM->hm.s.vmx.pRealModeTSS);
6521	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6522	if ( pVmcsInfo->fWasInRealMode
6523	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6524	{
6525	/*
6526	* Notify the recompiler must flush its code-cache as the guest -may-
6527	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6528	*/
6529	REMFlushTBs(pVM);
6530	Log4Func(("Switch to protected mode detected!\n"));
6531	pVmcsInfo->fWasInRealMode = false;
6532	}
6533	}
6534	#endif
6535	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6536	{
6537	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6538	if (pVmcsInfo->RealMode.fRealOnV86Active)
6539	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6540	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6541	AssertRCReturn(rc, rc);
6542	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6543	}
6544
6545	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6546	{
6547	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6548	if (pVmcsInfo->RealMode.fRealOnV86Active)
6549	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6550	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6551	AssertRCReturn(rc, rc);
6552	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6553	}
6554
6555	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6556	{
6557	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6558	if (pVmcsInfo->RealMode.fRealOnV86Active)
6559	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6560	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6561	AssertRCReturn(rc, rc);
6562	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6563	}
6564
6565	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6566	{
6567	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6568	if (pVmcsInfo->RealMode.fRealOnV86Active)
6569	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6570	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6571	AssertRCReturn(rc, rc);
6572	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6573	}
6574
6575	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6576	{
6577	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6578	if (pVmcsInfo->RealMode.fRealOnV86Active)
6579	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6580	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6581	AssertRCReturn(rc, rc);
6582	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6583	}
6584
6585	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6586	{
6587	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6588	if (pVmcsInfo->RealMode.fRealOnV86Active)
6589	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6590	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6591	AssertRCReturn(rc, rc);
6592	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6593	}
6594
6595	#ifdef VBOX_STRICT
6596	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6597	#endif
6598	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6599	pCtx->cs.Attr.u));
6600	}
6601
6602	/*
6603	* Guest TR.
6604	*/
6605	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6606	{
6607	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6608
6609	/*
6610	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6611	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6612	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6613	*/
6614	uint16_t u16Sel;
6615	uint32_t u32Limit;
6616	uint64_t u64Base;
6617	uint32_t u32AccessRights;
6618	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6619	{
6620	u16Sel = pCtx->tr.Sel;
6621	u32Limit = pCtx->tr.u32Limit;
6622	u64Base = pCtx->tr.u64Base;
6623	u32AccessRights = pCtx->tr.Attr.u;
6624	}
6625	else
6626	{
6627	Assert(!pVmxTransient->fIsNestedGuest);
6628	Assert(pVM->hm.s.vmx.pRealModeTSS);
6629	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6630
6631	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6632	RTGCPHYS GCPhys;
6633	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6634	AssertRCReturn(rc, rc);
6635
6636	X86DESCATTR DescAttr;
6637	DescAttr.u = 0;
6638	DescAttr.n.u1Present = 1;
6639	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6640
6641	u16Sel = 0;
6642	u32Limit = HM_VTX_TSS_SIZE;
6643	u64Base = GCPhys;
6644	u32AccessRights = DescAttr.u;
6645	}
6646
6647	/* Validate. */
6648	Assert(!(u16Sel & RT_BIT(2)));
6649	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6650	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6651	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6652	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6653	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6654	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6655	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6656	Assert( (u32Limit & 0xfff) == 0xfff
6657	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6658	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6659	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6660
6661	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6662	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6663	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6664	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6665	AssertRCReturn(rc, rc);
6666
6667	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6668	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6669	}
6670
6671	/*
6672	* Guest GDTR.
6673	*/
6674	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6675	{
6676	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6677
6678	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6679	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6680	AssertRCReturn(rc, rc);
6681
6682	/* Validate. */
6683	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6684
6685	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6686	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6687	}
6688
6689	/*
6690	* Guest LDTR.
6691	*/
6692	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6693	{
6694	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6695
6696	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6697	uint32_t u32Access;
6698	if ( !pVmxTransient->fIsNestedGuest
6699	&& !pCtx->ldtr.Attr.u)
6700	u32Access = X86DESCATTR_UNUSABLE;
6701	else
6702	u32Access = pCtx->ldtr.Attr.u;
6703
6704	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6705	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6706	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6707	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6708	AssertRCReturn(rc, rc);
6709
6710	/* Validate. */
6711	if (!(u32Access & X86DESCATTR_UNUSABLE))
6712	{
6713	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6714	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6715	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6716	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6717	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6718	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6719	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6720	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6721	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6722	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6723	}
6724
6725	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6726	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6727	}
6728
6729	/*
6730	* Guest IDTR.
6731	*/
6732	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6733	{
6734	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6735
6736	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6737	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6738	AssertRCReturn(rc, rc);
6739
6740	/* Validate. */
6741	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6742
6743	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6744	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6745	}
6746
6747	return VINF_SUCCESS;
6748	}
6749
6750
6751	/**
6752	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6753	* areas.
6754	*
6755	* These MSRs will automatically be loaded to the host CPU on every successful
6756	* VM-entry and stored from the host CPU on every successful VM-exit.
6757	*
6758	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6759	* actual host MSR values are not- updated here for performance reasons. See
6760	* hmR0VmxExportHostMsrs().
6761	*
6762	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6763	*
6764	* @returns VBox status code.
6765	* @param pVCpu The cross context virtual CPU structure.
6766	* @param pVmxTransient The VMX-transient structure.
6767	*
6768	* @remarks No-long-jump zone!!!
6769	*/
6770	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6771	{
6772	AssertPtr(pVCpu);
6773	AssertPtr(pVmxTransient);
6774
6775	PVM pVM = pVCpu->CTX_SUFF(pVM);
6776	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6777
6778	/*
6779	* MSRs that we use the auto-load/store MSR area in the VMCS.
6780	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6781	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6782	*
6783	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6784	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6785	* emulation, nothing to do here.
6786	*/
6787	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6788	{
6789	if ( !pVmxTransient->fIsNestedGuest
6790	&& pVM->hm.s.fAllow64BitGuests)
6791	{
6792	#if HC_ARCH_BITS == 32
6793	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6794	Assert(!pVmxTransient->fIsNestedGuest);
6795
6796	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6797	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6798	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6799	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6800	AssertRCReturn(rc, rc);
6801	#endif
6802	}
6803	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6804	}
6805
6806	/*
6807	* Guest Sysenter MSRs.
6808	*/
6809	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6810	{
6811	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6812
6813	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6814	{
6815	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6816	AssertRCReturn(rc, rc);
6817	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6818	}
6819
6820	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6821	{
6822	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6823	AssertRCReturn(rc, rc);
6824	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6825	}
6826
6827	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6828	{
6829	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6830	AssertRCReturn(rc, rc);
6831	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6832	}
6833	}
6834
6835	/*
6836	* Guest/host EFER MSR.
6837	*/
6838	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6839	{
6840	/* Whether we are using the VMCS to swap the EFER MSR must have been
6841	determined earlier while exporting VM-entry/VM-exit controls. */
6842	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6843	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6844
6845	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6846	{
6847	/*
6848	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6849	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6850	*/
6851	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6852	{
6853	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6854	AssertRCReturn(rc, rc);
6855	}
6856	else
6857	{
6858	/*
6859	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6860	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6861	*/
6862	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6863	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6864	AssertRCReturn(rc, rc);
6865	}
6866	}
6867	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6868	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6869
6870	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6871	}
6872
6873	/*
6874	* Other MSRs.
6875	* Speculation Control (R/W).
6876	*/
6877	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6878	{
6879	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6880	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6881	{
6882	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6883	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6884	AssertRCReturn(rc, rc);
6885	}
6886	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6887	}
6888
6889	return VINF_SUCCESS;
6890	}
6891
6892
6893	/**
6894	* Selects up the appropriate function to run guest code.
6895	*
6896	* @returns VBox status code.
6897	* @param pVCpu The cross context virtual CPU structure.
6898	* @param pVmxTransient The VMX-transient structure.
6899	*
6900	* @remarks No-long-jump zone!!!
6901	*/
6902	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6903	{
6904	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6905	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6906
6907	if (CPUMIsGuestInLongModeEx(pCtx))
6908	{
6909	#ifndef VBOX_ENABLE_64_BITS_GUESTS
6910	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6911	#endif
6912	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6913	#if HC_ARCH_BITS == 32
6914	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
6915	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
6916	{
6917	#ifdef VBOX_STRICT
6918	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6919	{
6920	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6921	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6922	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6923	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6924	("fCtxChanged=%#RX64\n", fCtxChanged));
6925	}
6926	#endif
6927	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
6928
6929	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
6930	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
6931	pVmcsInfo->fSwitchedTo64on32 = true;
6932	Log4Func(("Selected 64-bit switcher\n"));
6933	}
6934	#else
6935	/* 64-bit host. */
6936	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6937	#endif
6938	}
6939	else
6940	{
6941	/* Guest is not in long mode, use the 32-bit handler. */
6942	#if HC_ARCH_BITS == 32
6943	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
6944	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
6945	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
6946	{
6947	# ifdef VBOX_STRICT
6948	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
6949	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
6950	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
6951	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
6952	("fCtxChanged=%#RX64\n", fCtxChanged));
6953	# endif
6954	}
6955	# ifdef VBOX_ENABLE_64_BITS_GUESTS
6956	/*
6957	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
6958	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
6959	* switcher flag now because we know the guest is in a sane state where it's safe
6960	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
6961	* the much faster 32-bit switcher again.
6962	*/
6963	if (!pVmcsInfo->fSwitchedTo64on32)
6964	{
6965	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
6966	Log4Func(("Selected 32-bit switcher\n"));
6967	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6968	}
6969	else
6970	{
6971	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
6972	if ( pVmcsInfo->RealMode.fRealOnV86Active
6973	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
6974	{
6975	pVmcsInfo->fSwitchedTo64on32 = false;
6976	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6977	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
6978	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
6979	\| HM_CHANGED_HOST_CONTEXT);
6980	Log4Func(("Selected 32-bit switcher (safe)\n"));
6981	}
6982	}
6983	# else
6984	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6985	# endif
6986	#else
6987	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6988	#endif
6989	}
6990	Assert(pVmcsInfo->pfnStartVM);
6991	return VINF_SUCCESS;
6992	}
6993
6994
6995	/**
6996	* Wrapper for running the guest code in VT-x.
6997	*
6998	* @returns VBox status code, no informational status codes.
6999	* @param pVCpu The cross context virtual CPU structure.
7000	* @param pVmxTransient The VMX-transient structure.
7001	*
7002	* @remarks No-long-jump zone!!!
7003	*/
7004	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
7005	{
7006	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
7007	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7008	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
7009
7010	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
7011
7012	/*
7013	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
7014	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
7015	* callee-saved and thus the need for this XMM wrapper.
7016	*
7017	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
7018	*/
7019	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7020	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
7021	PVM pVM = pVCpu->CTX_SUFF(pVM);
7022	#ifdef VBOX_WITH_KERNEL_USING_XMM
7023	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
7024	#else
7025	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
7026	#endif
7027	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
7028	return rc;
7029	}
7030
7031
7032	/**
7033	* Reports world-switch error and dumps some useful debug info.
7034	*
7035	* @param pVCpu The cross context virtual CPU structure.
7036	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
7037	* @param pVmxTransient The VMX-transient structure (only
7038	* exitReason updated).
7039	*/
7040	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
7041	{
7042	Assert(pVCpu);
7043	Assert(pVmxTransient);
7044	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7045
7046	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
7047	switch (rcVMRun)
7048	{
7049	case VERR_VMX_INVALID_VMXON_PTR:
7050	AssertFailed();
7051	break;
7052	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
7053	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
7054	{
7055	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
7056	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
7057	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
7058	AssertRC(rc);
7059
7060	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7061	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
7062	Cannot do it here as we may have been long preempted. */
7063
7064	#ifdef VBOX_STRICT
7065	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7066	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
7067	pVmxTransient->uExitReason));
7068	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
7069	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
7070	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
7071	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
7072	else
7073	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
7074	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
7075	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
7076
7077	/* VMX control bits. */
7078	uint32_t u32Val;
7079	uint64_t u64Val;
7080	RTHCUINTREG uHCReg;
7081	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
7082	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
7083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
7084	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
7085	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
7086	{
7087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
7088	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
7089	}
7090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
7091	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
7092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
7093	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
7094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
7095	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
7096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
7097	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
7098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
7099	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
7100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
7101	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
7102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
7103	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
7104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
7105	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
7106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7107	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
7108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
7109	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
7110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
7111	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
7112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
7113	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
7114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
7115	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
7116	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7117	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7118	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7119	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7120	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7121	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7122	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7123	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7124	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7125	{
7126	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7127	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7128	}
7129
7130	/* Guest bits. */
7131	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
7132	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
7133	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
7134	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
7135	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
7136	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
7137	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
7138	{
7139	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
7140	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
7141	}
7142
7143	/* Host bits. */
7144	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
7145	Log4(("Host CR0 %#RHr\n", uHCReg));
7146	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
7147	Log4(("Host CR3 %#RHr\n", uHCReg));
7148	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
7149	Log4(("Host CR4 %#RHr\n", uHCReg));
7150
7151	RTGDTR HostGdtr;
7152	PCX86DESCHC pDesc;
7153	ASMGetGDTR(&HostGdtr);
7154	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
7155	Log4(("Host CS %#08x\n", u32Val));
7156	if (u32Val < HostGdtr.cbGdt)
7157	{
7158	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7159	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
7160	}
7161
7162	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
7163	Log4(("Host DS %#08x\n", u32Val));
7164	if (u32Val < HostGdtr.cbGdt)
7165	{
7166	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7167	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
7168	}
7169
7170	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
7171	Log4(("Host ES %#08x\n", u32Val));
7172	if (u32Val < HostGdtr.cbGdt)
7173	{
7174	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7175	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
7176	}
7177
7178	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
7179	Log4(("Host FS %#08x\n", u32Val));
7180	if (u32Val < HostGdtr.cbGdt)
7181	{
7182	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7183	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
7184	}
7185
7186	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
7187	Log4(("Host GS %#08x\n", u32Val));
7188	if (u32Val < HostGdtr.cbGdt)
7189	{
7190	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7191	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
7192	}
7193
7194	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
7195	Log4(("Host SS %#08x\n", u32Val));
7196	if (u32Val < HostGdtr.cbGdt)
7197	{
7198	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7199	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
7200	}
7201
7202	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
7203	Log4(("Host TR %#08x\n", u32Val));
7204	if (u32Val < HostGdtr.cbGdt)
7205	{
7206	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
7207	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
7208	}
7209
7210	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
7211	Log4(("Host TR Base %#RHv\n", uHCReg));
7212	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
7213	Log4(("Host GDTR Base %#RHv\n", uHCReg));
7214	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
7215	Log4(("Host IDTR Base %#RHv\n", uHCReg));
7216	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
7217	Log4(("Host SYSENTER CS %#08x\n", u32Val));
7218	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
7219	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
7220	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
7221	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
7222	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
7223	Log4(("Host RSP %#RHv\n", uHCReg));
7224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
7225	Log4(("Host RIP %#RHv\n", uHCReg));
7226	# if HC_ARCH_BITS == 64
7227	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
7228	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
7229	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
7230	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
7231	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
7232	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
7233	# endif
7234	#endif /* VBOX_STRICT */
7235	break;
7236	}
7237
7238	default:
7239	/* Impossible */
7240	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
7241	break;
7242	}
7243	}
7244
7245
7246	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
7247	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
7248	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
7249	# endif
7250
7251	/**
7252	* Initialize the VMCS-Read cache.
7253	*
7254	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
7255	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
7256	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
7257	* (those that have a 32-bit FULL & HIGH part).
7258	*
7259	* @param pVCpu The cross context virtual CPU structure.
7260	*/
7261	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
7262	{
7263	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
7264	do { \
7265	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
7266	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
7267	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
7268	++cReadFields; \
7269	} while (0)
7270
7271	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7272	uint32_t cReadFields = 0;
7273
7274	/*
7275	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
7276	* and serve to indicate exceptions to the rules.
7277	*/
7278
7279	/* Guest-natural selector base fields. */
7280	#if 0
7281	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
7282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
7283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
7284	#endif
7285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
7286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
7287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
7288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
7289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
7290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
7291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
7292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
7293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
7294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
7295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
7296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
7297	#if 0
7298	/* Unused natural width guest-state fields. */
7299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
7300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
7301	#endif
7302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
7303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
7304
7305	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
7306	these 64-bit fields (using "FULL" and "HIGH" fields). */
7307	#if 0
7308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
7309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
7310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
7311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
7312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
7313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
7314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
7315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
7316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
7317	#endif
7318
7319	/* Natural width guest-state fields. */
7320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
7321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
7322
7323	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
7324	{
7325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
7326	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
7327	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
7328	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
7329	}
7330	else
7331	{
7332	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
7333	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
7334	}
7335
7336	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
7337	}
7338
7339
7340	/**
7341	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
7342	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
7343	* darwin, running 64-bit guests).
7344	*
7345	* @returns VBox status code.
7346	* @param pVCpu The cross context virtual CPU structure.
7347	* @param idxField The VMCS field encoding.
7348	* @param u64Val 16, 32 or 64-bit value.
7349	*/
7350	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7351	{
7352	AssertPtr(pVCpu);
7353	int rc;
7354	switch (idxField)
7355	{
7356	/*
7357	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
7358	*/
7359	/* 64-bit Control fields. */
7360	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
7361	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
7362	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
7363	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
7364	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
7365	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
7366	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
7367	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
7368	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
7369	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
7370	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
7371	case VMX_VMCS64_CTRL_EPTP_FULL:
7372	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
7373	/* 64-bit Guest-state fields. */
7374	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
7375	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
7376	case VMX_VMCS64_GUEST_PAT_FULL:
7377	case VMX_VMCS64_GUEST_EFER_FULL:
7378	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
7379	case VMX_VMCS64_GUEST_PDPTE0_FULL:
7380	case VMX_VMCS64_GUEST_PDPTE1_FULL:
7381	case VMX_VMCS64_GUEST_PDPTE2_FULL:
7382	case VMX_VMCS64_GUEST_PDPTE3_FULL:
7383	/* 64-bit Host-state fields. */
7384	case VMX_VMCS64_HOST_PAT_FULL:
7385	case VMX_VMCS64_HOST_EFER_FULL:
7386	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
7387	{
7388	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7389	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
7390	break;
7391	}
7392
7393	/*
7394	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
7395	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
7396	*/
7397	/* Natural-width Guest-state fields. */
7398	case VMX_VMCS_GUEST_CR3:
7399	case VMX_VMCS_GUEST_ES_BASE:
7400	case VMX_VMCS_GUEST_CS_BASE:
7401	case VMX_VMCS_GUEST_SS_BASE:
7402	case VMX_VMCS_GUEST_DS_BASE:
7403	case VMX_VMCS_GUEST_FS_BASE:
7404	case VMX_VMCS_GUEST_GS_BASE:
7405	case VMX_VMCS_GUEST_LDTR_BASE:
7406	case VMX_VMCS_GUEST_TR_BASE:
7407	case VMX_VMCS_GUEST_GDTR_BASE:
7408	case VMX_VMCS_GUEST_IDTR_BASE:
7409	case VMX_VMCS_GUEST_RSP:
7410	case VMX_VMCS_GUEST_RIP:
7411	case VMX_VMCS_GUEST_SYSENTER_ESP:
7412	case VMX_VMCS_GUEST_SYSENTER_EIP:
7413	{
7414	if (!(RT_HI_U32(u64Val)))
7415	{
7416	/* If this field is 64-bit, VT-x will zero out the top bits. */
7417	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
7418	}
7419	else
7420	{
7421	/* Assert that only the 32->64 switcher case should ever come here. */
7422	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
7423	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
7424	}
7425	break;
7426	}
7427
7428	default:
7429	{
7430	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
7431	pVCpu->hm.s.u32HMError = idxField;
7432	rc = VERR_INVALID_PARAMETER;
7433	break;
7434	}
7435	}
7436	AssertRCReturn(rc, rc);
7437	return rc;
7438	}
7439
7440
7441	/**
7442	* Queue up a VMWRITE by using the VMCS write cache.
7443	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
7444	*
7445	* @param pVCpu The cross context virtual CPU structure.
7446	* @param idxField The VMCS field encoding.
7447	* @param u64Val 16, 32 or 64-bit value.
7448	*/
7449	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
7450	{
7451	AssertPtr(pVCpu);
7452	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
7453
7454	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
7455	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
7456
7457	/* Make sure there are no duplicates. */
7458	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
7459	{
7460	if (pCache->Write.aField[i] == idxField)
7461	{
7462	pCache->Write.aFieldVal[i] = u64Val;
7463	return VINF_SUCCESS;
7464	}
7465	}
7466
7467	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
7468	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
7469	pCache->Write.cValidEntries++;
7470	return VINF_SUCCESS;
7471	}
7472	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
7473
7474
7475	/**
7476	* Sets up the usage of TSC-offsetting and updates the VMCS.
7477	*
7478	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
7479	* VMX-preemption timer.
7480	*
7481	* @returns VBox status code.
7482	* @param pVCpu The cross context virtual CPU structure.
7483	* @param pVmxTransient The VMX-transient structure.
7484	*
7485	* @remarks No-long-jump zone!!!
7486	*/
7487	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
7488	{
7489	bool fOffsettedTsc;
7490	bool fParavirtTsc;
7491	uint64_t uTscOffset;
7492	PVM pVM = pVCpu->CTX_SUFF(pVM);
7493	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7494
7495	if (pVM->hm.s.vmx.fUsePreemptTimer)
7496	{
7497	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
7498
7499	/* Make sure the returned values have sane upper and lower boundaries. */
7500	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
7501	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
7502	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
7503	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
7504
7505	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
7506	* preemption timers here. We probably need to clamp the preemption timer,
7507	* after converting the timer value to the host. */
7508	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
7509	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
7510	AssertRC(rc);
7511	}
7512	else
7513	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
7514
7515	if (fParavirtTsc)
7516	{
7517	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
7518	information before every VM-entry, hence disable it for performance sake. */
7519	#if 0
7520	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7521	AssertRC(rc);
7522	#endif
7523	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7524	}
7525
7526	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
7527	if ( fOffsettedTsc
7528	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7529	{
7530	if (pVmxTransient->fIsNestedGuest)
7531	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7532	if (pVmcsInfo->u64TscOffset != uTscOffset)
7533	{
7534	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
7535	AssertRC(rc);
7536	pVmcsInfo->u64TscOffset = uTscOffset;
7537	}
7538
7539	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
7540	{
7541	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
7542	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7543	AssertRC(rc);
7544	pVmcsInfo->u32ProcCtls = uProcCtls;
7545	}
7546	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
7547	}
7548	else
7549	{
7550	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7551	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
7552	{
7553	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
7554	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
7555	AssertRC(rc);
7556	pVmcsInfo->u32ProcCtls = uProcCtls;
7557	}
7558	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
7559	}
7560	}
7561
7562
7563	/**
7564	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7565	* VM-exit interruption info type.
7566	*
7567	* @returns The IEM exception flags.
7568	* @param uVector The event vector.
7569	* @param uVmxEventType The VMX event type.
7570	*
7571	* @remarks This function currently only constructs flags required for
7572	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7573	* and CR2 aspects of an exception are not included).
7574	*/
7575	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7576	{
7577	uint32_t fIemXcptFlags;
7578	switch (uVmxEventType)
7579	{
7580	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7581	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7582	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7583	break;
7584
7585	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7586	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7587	break;
7588
7589	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7590	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7591	break;
7592
7593	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7594	{
7595	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7596	if (uVector == X86_XCPT_BP)
7597	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7598	else if (uVector == X86_XCPT_OF)
7599	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7600	else
7601	{
7602	fIemXcptFlags = 0;
7603	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7604	}
7605	break;
7606	}
7607
7608	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7609	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7610	break;
7611
7612	default:
7613	fIemXcptFlags = 0;
7614	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7615	break;
7616	}
7617	return fIemXcptFlags;
7618	}
7619
7620
7621	/**
7622	* Sets an event as a pending event to be injected into the guest.
7623	*
7624	* @param pVCpu The cross context virtual CPU structure.
7625	* @param u32IntInfo The VM-entry interruption-information field.
7626	* @param cbInstr The VM-entry instruction length in bytes (for software
7627	* interrupts, exceptions and privileged software
7628	* exceptions).
7629	* @param u32ErrCode The VM-entry exception error code.
7630	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7631	* page-fault.
7632	*/
7633	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7634	RTGCUINTPTR GCPtrFaultAddress)
7635	{
7636	Assert(!pVCpu->hm.s.Event.fPending);
7637	pVCpu->hm.s.Event.fPending = true;
7638	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7639	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7640	pVCpu->hm.s.Event.cbInstr = cbInstr;
7641	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7642	}
7643
7644
7645	/**
7646	* Sets an external interrupt as pending-for-injection into the VM.
7647	*
7648	* @param pVCpu The cross context virtual CPU structure.
7649	* @param u8Interrupt The external interrupt vector.
7650	*/
7651	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7652	{
7653	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7654	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7655	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7656	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7657	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7658	}
7659
7660
7661	/**
7662	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7663	*
7664	* @param pVCpu The cross context virtual CPU structure.
7665	*/
7666	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7667	{
7668	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7669	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7670	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7671	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7672	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7673	}
7674
7675
7676	/**
7677	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7678	*
7679	* @param pVCpu The cross context virtual CPU structure.
7680	*/
7681	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7682	{
7683	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7684	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7685	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7686	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7687	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7688	}
7689
7690
7691	/**
7692	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7693	*
7694	* @param pVCpu The cross context virtual CPU structure.
7695	*/
7696	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7697	{
7698	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7699	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7700	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7701	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7702	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7703	}
7704
7705
7706	/**
7707	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7708	*
7709	* @param pVCpu The cross context virtual CPU structure.
7710	*/
7711	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7712	{
7713	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7714	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7715	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7716	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7717	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7718	}
7719
7720
7721	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7722	/**
7723	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7724	*
7725	* @param pVCpu The cross context virtual CPU structure.
7726	* @param u32ErrCode The error code for the general-protection exception.
7727	*/
7728	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7729	{
7730	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7731	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7732	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7733	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7734	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7735	}
7736
7737
7738	/**
7739	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7740	*
7741	* @param pVCpu The cross context virtual CPU structure.
7742	* @param u32ErrCode The error code for the stack exception.
7743	*/
7744	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7745	{
7746	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7747	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7748	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7749	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7750	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7751	}
7752	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7753
7754
7755	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7756	{
7757	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7758
7759	/*
7760	* If VT-x marks the segment as unusable, most other bits remain undefined:
7761	* - For CS the L, D and G bits have meaning.
7762	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7763	* - For the remaining data segments no bits are defined.
7764	*
7765	* The present bit and the unusable bit has been observed to be set at the
7766	* same time (the selector was supposed to be invalid as we started executing
7767	* a V8086 interrupt in ring-0).
7768	*
7769	* What should be important for the rest of the VBox code, is that the P bit is
7770	* cleared. Some of the other VBox code recognizes the unusable bit, but
7771	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7772	* safe side here, we'll strip off P and other bits we don't care about. If
7773	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7774	*
7775	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7776	*/
7777	#ifdef VBOX_STRICT
7778	uint32_t const uAttr = pSelReg->Attr.u;
7779	#endif
7780
7781	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7782	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7783	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7784
7785	#ifdef VBOX_STRICT
7786	VMMRZCallRing3Disable(pVCpu);
7787	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7788	# ifdef DEBUG_bird
7789	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7790	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7791	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7792	# endif
7793	VMMRZCallRing3Enable(pVCpu);
7794	NOREF(uAttr);
7795	#endif
7796	RT_NOREF2(pVCpu, idxSel);
7797	}
7798
7799
7800	/**
7801	* Imports a guest segment register from the current VMCS into the guest-CPU
7802	* context.
7803	*
7804	* @returns VBox status code.
7805	* @param pVCpu The cross context virtual CPU structure.
7806	* @param iSegReg The segment register number (X86_SREG_XXX).
7807	*
7808	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7809	* do not log!
7810	*/
7811	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7812	{
7813	Assert(iSegReg < X86_SREG_COUNT);
7814
7815	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7816	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7817	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7818	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7819	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7820	#else
7821	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7822	#endif
7823	uint64_t u64Base;
7824	uint32_t u32Sel, u32Limit, u32Attr;
7825	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7826	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7827	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7828	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7829	if (RT_SUCCESS(rc))
7830	{
7831	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7832	pSelReg->Sel = u32Sel;
7833	pSelReg->ValidSel = u32Sel;
7834	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7835	pSelReg->u32Limit = u32Limit;
7836	pSelReg->u64Base = u64Base;
7837	pSelReg->Attr.u = u32Attr;
7838	if (u32Attr & X86DESCATTR_UNUSABLE)
7839	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7840	}
7841	return rc;
7842	}
7843
7844
7845	/**
7846	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7847	*
7848	* @returns VBox status code.
7849	* @param pVCpu The cross context virtual CPU structure.
7850	*
7851	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7852	* do not log!
7853	*/
7854	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7855	{
7856	uint64_t u64Base;
7857	uint32_t u32Sel, u32Limit, u32Attr;
7858	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7859	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7860	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7861	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7862	if (RT_SUCCESS(rc))
7863	{
7864	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7865	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7866	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7867	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7868	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7869	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7870	if (u32Attr & X86DESCATTR_UNUSABLE)
7871	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7872	}
7873	return rc;
7874	}
7875
7876
7877	/**
7878	* Imports the guest TR from the current VMCS into the guest-CPU context.
7879	*
7880	* @returns VBox status code.
7881	* @param pVCpu The cross context virtual CPU structure.
7882	*
7883	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7884	* do not log!
7885	*/
7886	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7887	{
7888	uint32_t u32Sel, u32Limit, u32Attr;
7889	uint64_t u64Base;
7890	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7891	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7892	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7893	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7894	AssertRCReturn(rc, rc);
7895
7896	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7897	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7898	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7899	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7900	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7901	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7902	/* TR is the only selector that can never be unusable. */
7903	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7904	return VINF_SUCCESS;
7905	}
7906
7907
7908	/**
7909	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7910	*
7911	* @returns VBox status code.
7912	* @param pVCpu The cross context virtual CPU structure.
7913	*
7914	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7915	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7916	* instead!!!
7917	*/
7918	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7919	{
7920	uint64_t u64Val;
7921	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7922	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7923	{
7924	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7925	if (RT_SUCCESS(rc))
7926	{
7927	pCtx->rip = u64Val;
7928	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7929	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7930	}
7931	return rc;
7932	}
7933	return VINF_SUCCESS;
7934	}
7935
7936
7937	/**
7938	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7939	*
7940	* @returns VBox status code.
7941	* @param pVCpu The cross context virtual CPU structure.
7942	* @param pVmcsInfo The VMCS info. object.
7943	*
7944	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7945	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7946	* instead!!!
7947	*/
7948	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7949	{
7950	uint32_t u32Val;
7951	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7952	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7953	{
7954	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7955	if (RT_SUCCESS(rc))
7956	{
7957	pCtx->eflags.u32 = u32Val;
7958
7959	/* Restore eflags for real-on-v86-mode hack. */
7960	if (pVmcsInfo->RealMode.fRealOnV86Active)
7961	{
7962	pCtx->eflags.Bits.u1VM = 0;
7963	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7964	}
7965	}
7966	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7967	return rc;
7968	}
7969	return VINF_SUCCESS;
7970	}
7971
7972
7973	/**
7974	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7975	* context.
7976	*
7977	* @returns VBox status code.
7978	* @param pVCpu The cross context virtual CPU structure.
7979	* @param pVmcsInfo The VMCS info. object.
7980	*
7981	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7982	* do not log!
7983	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7984	* instead!!!
7985	*/
7986	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7987	{
7988	uint32_t u32Val;
7989	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7990	if (RT_SUCCESS(rc))
7991	{
7992	if (!u32Val)
7993	{
7994	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7995	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7996
7997	CPUMSetGuestNmiBlocking(pVCpu, false);
7998	}
7999	else
8000	{
8001	/*
8002	* We must import RIP here to set our EM interrupt-inhibited state.
8003	* We also import RFLAGS as our code that evaluates pending interrupts
8004	* before VM-entry requires it.
8005	*/
8006	rc = hmR0VmxImportGuestRip(pVCpu);
8007	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8008	if (RT_SUCCESS(rc))
8009	{
8010	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
8011	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
8012	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
8013	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
8014
8015	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8016	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
8017	}
8018	}
8019	}
8020	return rc;
8021	}
8022
8023
8024	/**
8025	* Worker for VMXR0ImportStateOnDemand.
8026	*
8027	* @returns VBox status code.
8028	* @param pVCpu The cross context virtual CPU structure.
8029	* @param pVmcsInfo The VMCS info. object.
8030	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8031	*/
8032	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
8033	{
8034	#define VMXLOCAL_BREAK_RC(a_rc) \
8035	if (RT_SUCCESS(a_rc)) \
8036	{ } \
8037	else \
8038	break
8039
8040	int rc = VINF_SUCCESS;
8041	PVM pVM = pVCpu->CTX_SUFF(pVM);
8042	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8043	uint64_t u64Val;
8044	uint32_t u32Val;
8045
8046	/*
8047	* Note! This is hack to workaround a mysterious BSOD observed with release builds
8048	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
8049	* neither are other host platforms.
8050	*
8051	* Committing this temporarily as it prevents BSOD.
8052	*
8053	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
8054	*/
8055	#ifdef RT_OS_WINDOWS
8056	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
8057	return VERR_HM_IPE_1;
8058	#endif
8059
8060	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
8061
8062	/*
8063	* We disable interrupts to make the updating of the state and in particular
8064	* the fExtrn modification atomic wrt to preemption hooks.
8065	*/
8066	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
8067
8068	fWhat &= pCtx->fExtrn;
8069	if (fWhat)
8070	{
8071	do
8072	{
8073	if (fWhat & CPUMCTX_EXTRN_RIP)
8074	{
8075	rc = hmR0VmxImportGuestRip(pVCpu);
8076	VMXLOCAL_BREAK_RC(rc);
8077	}
8078
8079	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
8080	{
8081	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
8082	VMXLOCAL_BREAK_RC(rc);
8083	}
8084
8085	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
8086	{
8087	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
8088	VMXLOCAL_BREAK_RC(rc);
8089	}
8090
8091	if (fWhat & CPUMCTX_EXTRN_RSP)
8092	{
8093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
8094	VMXLOCAL_BREAK_RC(rc);
8095	pCtx->rsp = u64Val;
8096	}
8097
8098	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
8099	{
8100	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
8101	if (fWhat & CPUMCTX_EXTRN_CS)
8102	{
8103	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
8104	rc \|= hmR0VmxImportGuestRip(pVCpu);
8105	if (fRealOnV86Active)
8106	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
8107	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
8108	}
8109	if (fWhat & CPUMCTX_EXTRN_SS)
8110	{
8111	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
8112	if (fRealOnV86Active)
8113	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
8114	}
8115	if (fWhat & CPUMCTX_EXTRN_DS)
8116	{
8117	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
8118	if (fRealOnV86Active)
8119	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
8120	}
8121	if (fWhat & CPUMCTX_EXTRN_ES)
8122	{
8123	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
8124	if (fRealOnV86Active)
8125	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
8126	}
8127	if (fWhat & CPUMCTX_EXTRN_FS)
8128	{
8129	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
8130	if (fRealOnV86Active)
8131	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
8132	}
8133	if (fWhat & CPUMCTX_EXTRN_GS)
8134	{
8135	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
8136	if (fRealOnV86Active)
8137	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
8138	}
8139	VMXLOCAL_BREAK_RC(rc);
8140	}
8141
8142	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
8143	{
8144	if (fWhat & CPUMCTX_EXTRN_LDTR)
8145	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
8146
8147	if (fWhat & CPUMCTX_EXTRN_GDTR)
8148	{
8149	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8150	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8151	pCtx->gdtr.pGdt = u64Val;
8152	pCtx->gdtr.cbGdt = u32Val;
8153	}
8154
8155	/* Guest IDTR. */
8156	if (fWhat & CPUMCTX_EXTRN_IDTR)
8157	{
8158	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8159	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8160	pCtx->idtr.pIdt = u64Val;
8161	pCtx->idtr.cbIdt = u32Val;
8162	}
8163
8164	/* Guest TR. */
8165	if (fWhat & CPUMCTX_EXTRN_TR)
8166	{
8167	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
8168	don't need to import that one. */
8169	if (!pVmcsInfo->RealMode.fRealOnV86Active)
8170	rc \|= hmR0VmxImportGuestTr(pVCpu);
8171	}
8172	VMXLOCAL_BREAK_RC(rc);
8173	}
8174
8175	if (fWhat & CPUMCTX_EXTRN_DR7)
8176	{
8177	if (!pVCpu->hm.s.fUsingHyperDR7)
8178	{
8179	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
8180	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
8181	VMXLOCAL_BREAK_RC(rc);
8182	pCtx->dr[7] = u32Val;
8183	}
8184	}
8185
8186	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
8187	{
8188	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
8189	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
8190	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
8191	pCtx->SysEnter.cs = u32Val;
8192	VMXLOCAL_BREAK_RC(rc);
8193	}
8194
8195	#if HC_ARCH_BITS == 64
8196	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
8197	{
8198	if ( pVM->hm.s.fAllow64BitGuests
8199	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8200	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
8201	}
8202
8203	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
8204	{
8205	if ( pVM->hm.s.fAllow64BitGuests
8206	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
8207	{
8208	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
8209	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
8210	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
8211	}
8212	}
8213	#endif
8214
8215	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
8216	#if HC_ARCH_BITS == 32
8217	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
8218	#endif
8219	)
8220	{
8221	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
8222	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
8223	Assert(pMsrs);
8224	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
8225	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
8226	for (uint32_t i = 0; i < cMsrs; i++)
8227	{
8228	uint32_t const idMsr = pMsrs[i].u32Msr;
8229	switch (idMsr)
8230	{
8231	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
8232	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
8233	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
8234	#if HC_ARCH_BITS == 32
8235	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
8236	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
8237	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
8238	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
8239	#endif
8240	default:
8241	{
8242	pCtx->fExtrn = 0;
8243	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
8244	ASMSetFlags(fEFlags);
8245	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
8246	return VERR_HM_UNEXPECTED_LD_ST_MSR;
8247	}
8248	}
8249	}
8250	}
8251
8252	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
8253	{
8254	uint64_t u64Shadow;
8255	if (fWhat & CPUMCTX_EXTRN_CR0)
8256	{
8257	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8258	* remove when we drop 32-bit host w/ 64-bit host support, see
8259	* @bugref{9180#c39}. */
8260	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8261	#if HC_ARCH_BITS == 32
8262	uint32_t u32Shadow;
8263	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
8264	u64Shadow = u32Shadow;
8265	#else
8266	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
8267	#endif
8268	VMXLOCAL_BREAK_RC(rc);
8269	u64Val = u32Val;
8270	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
8271	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
8272	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8273	/*
8274	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
8275	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
8276	*/
8277	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8278	{
8279	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
8280	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
8281	}
8282	#endif
8283	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
8284	CPUMSetGuestCR0(pVCpu, u64Val);
8285	VMMRZCallRing3Enable(pVCpu);
8286	}
8287
8288	if (fWhat & CPUMCTX_EXTRN_CR4)
8289	{
8290	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
8291	* remove when we drop 32-bit host w/ 64-bit host support, see
8292	* @bugref{9180#c39}. */
8293	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8294	#if HC_ARCH_BITS == 32
8295	uint32_t u32Shadow;
8296	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
8297	u64Shadow = u32Shadow;
8298	#else
8299	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
8300	#endif
8301	VMXLOCAL_BREAK_RC(rc);
8302	u64Val = u32Val;
8303	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
8304	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
8305	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8306	/*
8307	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
8308	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
8309	*/
8310	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8311	{
8312	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
8313	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
8314	}
8315	#endif
8316	pCtx->cr4 = u64Val;
8317	}
8318
8319	if (fWhat & CPUMCTX_EXTRN_CR3)
8320	{
8321	/* CR0.PG bit changes are always intercepted, so it's up to date. */
8322	if ( pVM->hm.s.vmx.fUnrestrictedGuest
8323	\|\| ( pVM->hm.s.fNestedPaging
8324	&& CPUMIsGuestPagingEnabledEx(pCtx)))
8325	{
8326	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
8327	VMXLOCAL_BREAK_RC(rc);
8328	if (pCtx->cr3 != u64Val)
8329	{
8330	pCtx->cr3 = u64Val;
8331	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
8332	}
8333
8334	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
8335	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
8336	if (CPUMIsGuestInPAEModeEx(pCtx))
8337	{
8338	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
8339	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
8340	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
8341	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
8342	VMXLOCAL_BREAK_RC(rc);
8343	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
8344	}
8345	}
8346	}
8347	}
8348
8349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8350	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
8351	{
8352	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
8353	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
8354	{
8355	Assert(CPUMIsGuestInVmxRootMode(pCtx));
8356	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
8357	VMXLOCAL_BREAK_RC(rc);
8358	}
8359
8360	# if 0
8361	/** @todo NSTVMX: We handle most of these fields individually by passing it to IEM
8362	* VM-exit handlers as parameters. We would handle it differently when using
8363	* the fast path. */
8364	/*
8365	* The hardware virtualization state currently consists of VMCS fields that may be
8366	* modified by execution of the nested-guest (that are not part of the general
8367	* guest state) and is visible to guest software. Hence, it is technically part of
8368	* the guest-CPU state when executing a nested-guest.
8369	*/
8370	if (CPUMIsGuestInVmxNonRootMode(pCtx))
8371	{
8372	PVMXVVMCS pGstVmcs = pCtx->hwvirt.vmx.CTX_SUFF(pVmcs);
8373	rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pGstVmcs->u32RoExitReason);
8374	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pGstVmcs->u64RoExitQual.u);
8375	VMXLOCAL_BREAK_RC(rc);
8376
8377	/*
8378	* VM-entry can fail due to invalid-guest state, machine-check events and
8379	* MSR loading failures. Other than VM-exit reason and Exit qualification
8380	* all other VMCS fields are left unmodified on VM-entry failure.
8381	*
8382	* See Intel spec. 26.7 "VM-entry Failures During Or After Loading Guest State".
8383	*/
8384	bool const fEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(pGstVmcs->u32RoExitReason);
8385	if (!fEntryFailed)
8386	{
8387	/*
8388	* Some notes on VMCS fields that may need importing when the fast path
8389	* is implemented. Currently we fully emulate VMLAUNCH/VMRESUME in IEM.
8390	*
8391	* Requires fixing up when using hardware-assisted VMX:
8392	* - VM-exit interruption info: Shouldn't reflect host interrupts/NMIs.
8393	* - VM-exit interruption error code: Cleared to 0 when not appropriate.
8394	* - IDT-vectoring info: Think about this.
8395	* - IDT-vectoring error code: Think about this.
8396	*
8397	* Emulated:
8398	* - Guest-interruptiblity state: Derived from FFs and RIP.
8399	* - Guest pending debug exceptions: Derived from DR6.
8400	* - Guest activity state: Emulated from EM state.
8401	* - Guest PDPTEs: Currently all 0s since we don't support nested EPT.
8402	* - Entry-interrupt info: Emulated, cleared to 0.
8403	*/
8404	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pGstVmcs->u32RoExitIntInfo);
8405	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pGstVmcs->u32RoExitIntErrCode);
8406	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pGstVmcs->u32RoIdtVectoringInfo);
8407	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pGstVmcs->u32RoIdtVectoringErrCode);
8408	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pGstVmcs->u32RoExitInstrLen);
8409	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pGstVmcs->u32RoExitIntInfo);
8410	rc \|= VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pGstVmcs->u64RoGuestPhysAddr.u);
8411	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pGstVmcs->u64RoGuestLinearAddr.u);
8412	/** @todo NSTVMX: Save and adjust preemption timer value. */
8413	}
8414
8415	VMXLOCAL_BREAK_RC(rc);
8416	}
8417	# endif
8418	}
8419	#endif
8420	} while (0);
8421
8422	if (RT_SUCCESS(rc))
8423	{
8424	/* Update fExtrn. */
8425	pCtx->fExtrn &= ~fWhat;
8426
8427	/* If everything has been imported, clear the HM keeper bit. */
8428	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
8429	{
8430	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
8431	Assert(!pCtx->fExtrn);
8432	}
8433	}
8434	}
8435	else
8436	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
8437
8438	ASMSetFlags(fEFlags);
8439
8440	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
8441
8442	if (RT_SUCCESS(rc))
8443	{ /* likely */ }
8444	else
8445	return rc;
8446
8447	/*
8448	* Honor any pending CR3 updates.
8449	*
8450	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
8451	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
8452	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
8453	*
8454	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
8455	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
8456	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
8457	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
8458	*
8459	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
8460	*/
8461	if (VMMRZCallRing3IsEnabled(pVCpu))
8462	{
8463	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8464	{
8465	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8466	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8467	}
8468
8469	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8470	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8471
8472	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8473	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8474	}
8475
8476	return VINF_SUCCESS;
8477	#undef VMXLOCAL_BREAK_RC
8478	}
8479
8480
8481	/**
8482	* Saves the guest state from the VMCS into the guest-CPU context.
8483	*
8484	* @returns VBox status code.
8485	* @param pVCpu The cross context virtual CPU structure.
8486	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
8487	*/
8488	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
8489	{
8490	AssertPtr(pVCpu);
8491	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8492	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
8493	}
8494
8495
8496	/**
8497	* Check per-VM and per-VCPU force flag actions that require us to go back to
8498	* ring-3 for one reason or another.
8499	*
8500	* @returns Strict VBox status code (i.e. informational status codes too)
8501	* @retval VINF_SUCCESS if we don't have any actions that require going back to
8502	* ring-3.
8503	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
8504	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
8505	* interrupts)
8506	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
8507	* all EMTs to be in ring-3.
8508	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
8509	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
8510	* to the EM loop.
8511	*
8512	* @param pVCpu The cross context virtual CPU structure.
8513	* @param fStepping Whether we are single-stepping the guest using the
8514	* hypervisor debugger.
8515	*/
8516	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
8517	{
8518	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8519
8520	/*
8521	* Update pending interrupts into the APIC's IRR.
8522	*/
8523	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
8524	APICUpdatePendingInterrupts(pVCpu);
8525
8526	/*
8527	* Anything pending? Should be more likely than not if we're doing a good job.
8528	*/
8529	PVM pVM = pVCpu->CTX_SUFF(pVM);
8530	if ( !fStepping
8531	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
8532	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
8533	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
8534	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
8535	return VINF_SUCCESS;
8536
8537	/* Pending PGM C3 sync. */
8538	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
8539	{
8540	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8541	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
8542	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
8543	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
8544	if (rcStrict2 != VINF_SUCCESS)
8545	{
8546	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
8547	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
8548	return rcStrict2;
8549	}
8550	}
8551
8552	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
8553	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
8554	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8555	{
8556	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8557	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
8558	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
8559	return rc2;
8560	}
8561
8562	/* Pending VM request packets, such as hardware interrupts. */
8563	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
8564	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
8565	{
8566	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
8567	Log4Func(("Pending VM request forcing us back to ring-3\n"));
8568	return VINF_EM_PENDING_REQUEST;
8569	}
8570
8571	/* Pending PGM pool flushes. */
8572	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
8573	{
8574	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
8575	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
8576	return VINF_PGM_POOL_FLUSH_PENDING;
8577	}
8578
8579	/* Pending DMA requests. */
8580	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
8581	{
8582	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
8583	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
8584	return VINF_EM_RAW_TO_R3;
8585	}
8586
8587	return VINF_SUCCESS;
8588	}
8589
8590
8591	/**
8592	* Converts any TRPM trap into a pending HM event. This is typically used when
8593	* entering from ring-3 (not longjmp returns).
8594	*
8595	* @param pVCpu The cross context virtual CPU structure.
8596	*/
8597	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8598	{
8599	Assert(TRPMHasTrap(pVCpu));
8600	Assert(!pVCpu->hm.s.Event.fPending);
8601
8602	uint8_t uVector;
8603	TRPMEVENT enmTrpmEvent;
8604	RTGCUINT uErrCode;
8605	RTGCUINTPTR GCPtrFaultAddress;
8606	uint8_t cbInstr;
8607
8608	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8609	AssertRC(rc);
8610
8611	uint32_t u32IntInfo;
8612	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8613	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
8614
8615	rc = TRPMResetTrap(pVCpu);
8616	AssertRC(rc);
8617	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8618	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8619
8620	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8621	}
8622
8623
8624	/**
8625	* Converts the pending HM event into a TRPM trap.
8626	*
8627	* @param pVCpu The cross context virtual CPU structure.
8628	*/
8629	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8630	{
8631	Assert(pVCpu->hm.s.Event.fPending);
8632
8633	/* If a trap was already pending, we did something wrong! */
8634	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8635
8636	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8637	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8638	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8639
8640	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8641
8642	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8643	AssertRC(rc);
8644
8645	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8646	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8647
8648	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8649	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8650	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
8651	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8652
8653	/* We're now done converting the pending event. */
8654	pVCpu->hm.s.Event.fPending = false;
8655	}
8656
8657
8658	/**
8659	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8660	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8661	*
8662	* @param pVCpu The cross context virtual CPU structure.
8663	* @param pVmcsInfo The VMCS info. object.
8664	*/
8665	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8666	{
8667	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8668	{
8669	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8670	{
8671	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8672	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8673	AssertRC(rc);
8674	}
8675	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8676	}
8677
8678
8679	/**
8680	* Clears the interrupt-window exiting control in the VMCS.
8681	*
8682	* @param pVmcsInfo The VMCS info. object.
8683	*/
8684	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8685	{
8686	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8687	{
8688	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8689	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8690	}
8691	return VINF_SUCCESS;
8692	}
8693
8694
8695	/**
8696	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8697	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8698	*
8699	* @param pVCpu The cross context virtual CPU structure.
8700	* @param pVmcsInfo The VMCS info. object.
8701	*/
8702	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8703	{
8704	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8705	{
8706	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8707	{
8708	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8709	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8710	AssertRC(rc);
8711	Log4Func(("Setup NMI-window exiting\n"));
8712	}
8713	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8714	}
8715
8716
8717	/**
8718	* Clears the NMI-window exiting control in the VMCS.
8719	*
8720	* @param pVmcsInfo The VMCS info. object.
8721	*/
8722	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8723	{
8724	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8725	{
8726	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8727	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8728	}
8729	return VINF_SUCCESS;
8730	}
8731
8732
8733	/**
8734	* Does the necessary state syncing before returning to ring-3 for any reason
8735	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8736	*
8737	* @returns VBox status code.
8738	* @param pVCpu The cross context virtual CPU structure.
8739	* @param fImportState Whether to import the guest state from the VMCS back
8740	* to the guest-CPU context.
8741	*
8742	* @remarks No-long-jmp zone!!!
8743	*/
8744	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8745	{
8746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8747	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8748
8749	RTCPUID const idCpu = RTMpCpuId();
8750	Log4Func(("HostCpuId=%u\n", idCpu));
8751
8752	/*
8753	* !!! IMPORTANT !!!
8754	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8755	*/
8756
8757	/* Save the guest state if necessary. */
8758	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8759	if (fImportState)
8760	{
8761	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8762	AssertRCReturn(rc, rc);
8763	}
8764
8765	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8766	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8767	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8768
8769	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8770	#ifdef VBOX_STRICT
8771	if (CPUMIsHyperDebugStateActive(pVCpu))
8772	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8773	#endif
8774	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8775	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8776	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8777
8778	#if HC_ARCH_BITS == 64
8779	/* Restore host-state bits that VT-x only restores partially. */
8780	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8781	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8782	{
8783	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8784	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8785	}
8786	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8787	#endif
8788
8789	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8790	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8791	{
8792	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8793	if (!fImportState)
8794	{
8795	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8796	AssertRCReturn(rc, rc);
8797	}
8798	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8799	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8800	}
8801	else
8802	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8803
8804	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8805	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8806
8807	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8808	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8809	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8810	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8811	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8812	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8813	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8814	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8815	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8816	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8817
8818	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8819
8820	/** @todo This partially defeats the purpose of having preemption hooks.
8821	* The problem is, deregistering the hooks should be moved to a place that
8822	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8823	* context.
8824	*/
8825	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8826	AssertRCReturn(rc, rc);
8827
8828	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8829	/*
8830	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8831	* clear a shadow VMCS before allowing that VMCS to become active on another
8832	* logical processor. We may or may not be importing guest state which clears
8833	* it, so cover for it here.
8834	*
8835	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8836	*/
8837	if ( pVmcsInfo->pvShadowVmcs
8838	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8839	{
8840	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8841	AssertRCReturn(rc, rc);
8842	}
8843
8844	/*
8845	* Flag that we need to re-import the host state if we switch to this VMCS before
8846	* executing guest or nested-guest code.
8847	*/
8848	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8849	#endif
8850
8851	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8852	NOREF(idCpu);
8853	return VINF_SUCCESS;
8854	}
8855
8856
8857	/**
8858	* Leaves the VT-x session.
8859	*
8860	* @returns VBox status code.
8861	* @param pVCpu The cross context virtual CPU structure.
8862	*
8863	* @remarks No-long-jmp zone!!!
8864	*/
8865	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8866	{
8867	HM_DISABLE_PREEMPT(pVCpu);
8868	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8869	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8870	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8871
8872	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8873	and done this from the VMXR0ThreadCtxCallback(). */
8874	if (!pVCpu->hm.s.fLeaveDone)
8875	{
8876	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8877	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8878	pVCpu->hm.s.fLeaveDone = true;
8879	}
8880	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8881
8882	/*
8883	* !!! IMPORTANT !!!
8884	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8885	*/
8886
8887	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8888	/** @todo Deregistering here means we need to VMCLEAR always
8889	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8890	* for calling VMMR0ThreadCtxHookDisable here! */
8891	VMMR0ThreadCtxHookDisable(pVCpu);
8892
8893	/* Leave HM context. This takes care of local init (term). */
8894	int rc = HMR0LeaveCpu(pVCpu);
8895
8896	HM_RESTORE_PREEMPT();
8897	return rc;
8898	}
8899
8900
8901	/**
8902	* Does the necessary state syncing before doing a longjmp to ring-3.
8903	*
8904	* @returns VBox status code.
8905	* @param pVCpu The cross context virtual CPU structure.
8906	*
8907	* @remarks No-long-jmp zone!!!
8908	*/
8909	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8910	{
8911	return hmR0VmxLeaveSession(pVCpu);
8912	}
8913
8914
8915	/**
8916	* Take necessary actions before going back to ring-3.
8917	*
8918	* An action requires us to go back to ring-3. This function does the necessary
8919	* steps before we can safely return to ring-3. This is not the same as longjmps
8920	* to ring-3, this is voluntary and prepares the guest so it may continue
8921	* executing outside HM (recompiler/IEM).
8922	*
8923	* @returns VBox status code.
8924	* @param pVCpu The cross context virtual CPU structure.
8925	* @param rcExit The reason for exiting to ring-3. Can be
8926	* VINF_VMM_UNKNOWN_RING3_CALL.
8927	*/
8928	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8929	{
8930	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8931
8932	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8933	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8934	{
8935	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8936	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8937	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8938	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8939	}
8940
8941	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8942	VMMRZCallRing3Disable(pVCpu);
8943	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8944
8945	/*
8946	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8947	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8948	*
8949	* This is because execution may continue from ring-3 and we would need to inject
8950	* the event from there (hence place it back in TRPM).
8951	*/
8952	if (pVCpu->hm.s.Event.fPending)
8953	{
8954	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8955	Assert(!pVCpu->hm.s.Event.fPending);
8956
8957	/* Clear the events from the VMCS. */
8958	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8959	AssertRCReturn(rc, rc);
8960	}
8961	#ifdef VBOX_STRICT
8962	else
8963	{
8964	/*
8965	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8966	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8967	* occasionally, see @bugref{9180#c42}.
8968	*
8969	* However, if the VM-entry failed, any VM entry-interruption info. field would
8970	* be left unmodified as the event would not have been injected to the guest. In
8971	* such cases, don't assert, we're not going to continue guest execution anyway.
8972	*/
8973	uint32_t uExitReason;
8974	uint32_t uEntryIntInfo;
8975	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8976	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8977	AssertRC(rc);
8978	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8979	}
8980	#endif
8981
8982	/*
8983	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8984	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8985	* (e.g. TPR below threshold).
8986	*/
8987	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8988	{
8989	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8990	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8991	AssertRCReturn(rc, rc);
8992	}
8993
8994	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8995	and if we're injecting an event we should have a TRPM trap pending. */
8996	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8997	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8998	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8999	#endif
9000
9001	/* Save guest state and restore host state bits. */
9002	int rc = hmR0VmxLeaveSession(pVCpu);
9003	AssertRCReturn(rc, rc);
9004	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
9005
9006	/* Thread-context hooks are unregistered at this point!!! */
9007
9008	/* Sync recompiler state. */
9009	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
9010	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
9011	\| CPUM_CHANGED_LDTR
9012	\| CPUM_CHANGED_GDTR
9013	\| CPUM_CHANGED_IDTR
9014	\| CPUM_CHANGED_TR
9015	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
9016	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
9017	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
9018	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
9019
9020	Assert(!pVCpu->hm.s.fClearTrapFlag);
9021
9022	/* Update the exit-to-ring 3 reason. */
9023	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
9024
9025	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
9026	if ( rcExit != VINF_EM_RAW_INTERRUPT
9027	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9028	{
9029	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
9030	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9031	}
9032
9033	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
9034
9035	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
9036	VMMRZCallRing3RemoveNotification(pVCpu);
9037	VMMRZCallRing3Enable(pVCpu);
9038
9039	return rc;
9040	}
9041
9042
9043	/**
9044	* VMMRZCallRing3() callback wrapper which saves the guest state before we
9045	* longjump to ring-3 and possibly get preempted.
9046	*
9047	* @returns VBox status code.
9048	* @param pVCpu The cross context virtual CPU structure.
9049	* @param enmOperation The operation causing the ring-3 longjump.
9050	* @param pvUser User argument, currently unused, NULL.
9051	*/
9052	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
9053	{
9054	RT_NOREF(pvUser);
9055	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
9056	{
9057	/*
9058	* !!! IMPORTANT !!!
9059	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
9060	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
9061	*/
9062	VMMRZCallRing3RemoveNotification(pVCpu);
9063	VMMRZCallRing3Disable(pVCpu);
9064	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
9065	RTThreadPreemptDisable(&PreemptState);
9066
9067	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9068	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9069	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
9070	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
9071
9072	#if HC_ARCH_BITS == 64
9073	/* Restore host-state bits that VT-x only restores partially. */
9074	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
9075	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
9076	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
9077	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
9078	#endif
9079
9080	/* Restore the lazy host MSRs as we're leaving VT-x context. */
9081	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
9082	hmR0VmxLazyRestoreHostMsrs(pVCpu);
9083
9084	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
9085	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
9086	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
9087
9088	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
9089	cleared as part of importing the guest state above. */
9090	hmR0VmxClearVmcs(pVmcsInfo);
9091
9092	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
9093	VMMR0ThreadCtxHookDisable(pVCpu);
9094	HMR0LeaveCpu(pVCpu);
9095	RTThreadPreemptRestore(&PreemptState);
9096	return VINF_SUCCESS;
9097	}
9098
9099	Assert(pVCpu);
9100	Assert(pvUser);
9101	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9102	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9103
9104	VMMRZCallRing3Disable(pVCpu);
9105	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9106
9107	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
9108
9109	int rc = hmR0VmxLongJmpToRing3(pVCpu);
9110	AssertRCReturn(rc, rc);
9111
9112	VMMRZCallRing3Enable(pVCpu);
9113	return VINF_SUCCESS;
9114	}
9115
9116
9117	/**
9118	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
9119	* stack.
9120	*
9121	* @returns Strict VBox status code (i.e. informational status codes too).
9122	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
9123	* @param pVCpu The cross context virtual CPU structure.
9124	* @param uValue The value to push to the guest stack.
9125	*/
9126	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
9127	{
9128	/*
9129	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
9130	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
9131	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
9132	*/
9133	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9134	if (pCtx->sp == 1)
9135	return VINF_EM_RESET;
9136	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
9137	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
9138	AssertRC(rc);
9139	return rc;
9140	}
9141
9142
9143	/**
9144	* Injects an event into the guest upon VM-entry by updating the relevant fields
9145	* in the VM-entry area in the VMCS.
9146	*
9147	* @returns Strict VBox status code (i.e. informational status codes too).
9148	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
9149	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
9150	*
9151	* @param pVCpu The cross context virtual CPU structure.
9152	* @param pVmxTransient The VMX-transient structure.
9153	* @param pEvent The event being injected.
9154	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
9155	* will be updated if necessary. This cannot not be NULL.
9156	* @param fStepping Whether we're single-stepping guest execution and should
9157	* return VINF_EM_DBG_STEPPED if the event is injected
9158	* directly (registers modified by us, not by hardware on
9159	* VM-entry).
9160	*/
9161	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
9162	uint32_t *pfIntrState)
9163	{
9164	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
9165	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
9166	Assert(pfIntrState);
9167
9168	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9169	uint32_t u32IntInfo = pEvent->u64IntInfo;
9170	uint32_t const u32ErrCode = pEvent->u32ErrCode;
9171	uint32_t const cbInstr = pEvent->cbInstr;
9172	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
9173	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
9174	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
9175
9176	#ifdef VBOX_STRICT
9177	/*
9178	* Validate the error-code-valid bit for hardware exceptions.
9179	* No error codes for exceptions in real-mode.
9180	*
9181	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9182	*/
9183	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9184	&& !CPUMIsGuestInRealModeEx(pCtx))
9185	{
9186	switch (uVector)
9187	{
9188	case X86_XCPT_PF:
9189	case X86_XCPT_DF:
9190	case X86_XCPT_TS:
9191	case X86_XCPT_NP:
9192	case X86_XCPT_SS:
9193	case X86_XCPT_GP:
9194	case X86_XCPT_AC:
9195	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
9196	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
9197	RT_FALL_THRU();
9198	default:
9199	break;
9200	}
9201	}
9202
9203	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
9204	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
9205	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
9206	#endif
9207
9208	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
9209
9210	/*
9211	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
9212	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
9213	* interrupt handler in the (real-mode) guest.
9214	*
9215	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
9216	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
9217	*/
9218	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
9219	{
9220	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
9221	{
9222	/*
9223	* For CPUs with unrestricted guest execution enabled and with the guest
9224	* in real-mode, we must not set the deliver-error-code bit.
9225	*
9226	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
9227	*/
9228	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
9229	}
9230	else
9231	{
9232	PVM pVM = pVCpu->CTX_SUFF(pVM);
9233	Assert(PDMVmmDevHeapIsEnabled(pVM));
9234	Assert(pVM->hm.s.vmx.pRealModeTSS);
9235	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
9236
9237	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
9238	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9239	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
9240	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
9241	AssertRCReturn(rc2, rc2);
9242
9243	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
9244	size_t const cbIdtEntry = sizeof(X86IDTR16);
9245	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
9246	{
9247	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
9248	if (uVector == X86_XCPT_DF)
9249	return VINF_EM_RESET;
9250
9251	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
9252	No error codes for exceptions in real-mode. */
9253	if (uVector == X86_XCPT_GP)
9254	{
9255	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
9256	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9257	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9258	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9259	HMEVENT EventXcptDf;
9260	RT_ZERO(EventXcptDf);
9261	EventXcptDf.u64IntInfo = uXcptDfInfo;
9262	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
9263	}
9264
9265	/*
9266	* If we're injecting an event with no valid IDT entry, inject a #GP.
9267	* No error codes for exceptions in real-mode.
9268	*
9269	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
9270	*/
9271	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
9272	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
9273	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
9274	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
9275	HMEVENT EventXcptGp;
9276	RT_ZERO(EventXcptGp);
9277	EventXcptGp.u64IntInfo = uXcptGpInfo;
9278	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
9279	}
9280
9281	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
9282	uint16_t uGuestIp = pCtx->ip;
9283	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
9284	{
9285	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9286	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
9287	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9288	}
9289	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
9290	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
9291
9292	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
9293	X86IDTR16 IdtEntry;
9294	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
9295	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
9296	AssertRCReturn(rc2, rc2);
9297
9298	/* Construct the stack frame for the interrupt/exception handler. */
9299	VBOXSTRICTRC rcStrict;
9300	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
9301	if (rcStrict == VINF_SUCCESS)
9302	{
9303	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
9304	if (rcStrict == VINF_SUCCESS)
9305	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
9306	}
9307
9308	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
9309	if (rcStrict == VINF_SUCCESS)
9310	{
9311	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
9312	pCtx->rip = IdtEntry.offSel;
9313	pCtx->cs.Sel = IdtEntry.uSel;
9314	pCtx->cs.ValidSel = IdtEntry.uSel;
9315	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
9316	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
9317	&& uVector == X86_XCPT_PF)
9318	pCtx->cr2 = GCPtrFault;
9319
9320	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
9321	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
9322	\| HM_CHANGED_GUEST_RSP);
9323
9324	/*
9325	* If we delivered a hardware exception (other than an NMI) and if there was
9326	* block-by-STI in effect, we should clear it.
9327	*/
9328	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9329	{
9330	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
9331	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
9332	Log4Func(("Clearing inhibition due to STI\n"));
9333	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
9334	}
9335
9336	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
9337	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
9338
9339	/*
9340	* The event has been truly dispatched to the guest. Mark it as no longer pending so
9341	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
9342	*/
9343	pVCpu->hm.s.Event.fPending = false;
9344
9345	/*
9346	* If we eventually support nested-guest execution without unrestricted guest execution,
9347	* we should clear fInterceptEvents here.
9348	*/
9349	Assert(!pVmxTransient->fIsNestedGuest);
9350
9351	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
9352	if (fStepping)
9353	rcStrict = VINF_EM_DBG_STEPPED;
9354	}
9355	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9356	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9357	return rcStrict;
9358	}
9359	}
9360
9361	/*
9362	* Validate.
9363	*/
9364	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
9365	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
9366
9367	/*
9368	* Inject the event into the VMCS.
9369	*/
9370	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
9371	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
9372	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
9373	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
9374	AssertRCReturn(rc, rc);
9375
9376	/*
9377	* Update guest CR2 if this is a page-fault.
9378	*/
9379	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
9380	&& uVector == X86_XCPT_PF)
9381	pCtx->cr2 = GCPtrFault;
9382
9383	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
9384	return VINF_SUCCESS;
9385	}
9386
9387
9388	/**
9389	* Evaluates the event to be delivered to the guest and sets it as the pending
9390	* event.
9391	*
9392	* @returns Strict VBox status code (i.e. informational status codes too).
9393	* @param pVCpu The cross context virtual CPU structure.
9394	* @param pVmxTransient The VMX-transient structure.
9395	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
9396	*/
9397	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
9398	{
9399	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9400	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9401	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
9402
9403	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
9404	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
9405	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9406	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9407	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9408
9409	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9410	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9411	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9412	Assert(!TRPMHasTrap(pVCpu));
9413	Assert(pfIntrState);
9414
9415	*pfIntrState = fIntrState;
9416
9417	/*
9418	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
9419	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
9420	*/
9421	/** @todo SMI. SMIs take priority over NMIs. */
9422	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
9423	{
9424	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
9425	if ( !pVCpu->hm.s.Event.fPending
9426	&& !fBlockNmi
9427	&& !fBlockSti
9428	&& !fBlockMovSS)
9429	{
9430	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9431	if ( fIsNestedGuest
9432	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
9433	return IEMExecVmxVmexitXcptNmi(pVCpu);
9434	#endif
9435	hmR0VmxSetPendingXcptNmi(pVCpu);
9436	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
9437	Log4Func(("Pending NMI\n"));
9438	}
9439	else if (!fIsNestedGuest)
9440	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
9441	}
9442	/*
9443	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
9444	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
9445	*/
9446	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
9447	&& !pVCpu->hm.s.fSingleInstruction)
9448	{
9449	Assert(!DBGFIsStepping(pVCpu));
9450	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
9451	AssertRCReturn(rc, rc);
9452	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
9453	if ( !pVCpu->hm.s.Event.fPending
9454	&& !fBlockInt
9455	&& !fBlockSti
9456	&& !fBlockMovSS)
9457	{
9458	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9459	if ( fIsNestedGuest
9460	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
9461	{
9462	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
9463	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9464	return rcStrict;
9465	}
9466	#endif
9467	uint8_t u8Interrupt;
9468	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
9469	if (RT_SUCCESS(rc))
9470	{
9471	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9472	if ( fIsNestedGuest
9473	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
9474	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
9475	{
9476	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
9477	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
9478	return rcStrict;
9479	}
9480	#endif
9481	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
9482	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
9483	}
9484	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
9485	{
9486	if ( !fIsNestedGuest
9487	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
9488	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
9489	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
9490
9491	/*
9492	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
9493	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
9494	* need to re-set this force-flag here.
9495	*/
9496	}
9497	else
9498	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
9499	}
9500	else if (!fIsNestedGuest)
9501	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
9502	}
9503
9504	return VINF_SUCCESS;
9505	}
9506
9507
9508	/**
9509	* Injects any pending events into the guest if the guest is in a state to
9510	* receive them.
9511	*
9512	* @returns Strict VBox status code (i.e. informational status codes too).
9513	* @param pVCpu The cross context virtual CPU structure.
9514	* @param pVmxTransient The VMX-transient structure.
9515	* @param fIntrState The VT-x guest-interruptibility state.
9516	* @param fStepping Whether we are single-stepping the guest using the
9517	* hypervisor debugger and should return
9518	* VINF_EM_DBG_STEPPED if the event was dispatched
9519	* directly.
9520	*/
9521	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
9522	{
9523	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9524	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9525
9526	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
9527	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
9528
9529	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
9530	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
9531	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
9532	Assert(!TRPMHasTrap(pVCpu));
9533
9534	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
9535	if (pVCpu->hm.s.Event.fPending)
9536	{
9537	/*
9538	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
9539	* pending even while injecting an event and in this case, we want a VM-exit as soon as
9540	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
9541	*
9542	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9543	*/
9544	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
9545	#ifdef VBOX_STRICT
9546	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9547	{
9548	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
9549	Assert(!fBlockInt);
9550	Assert(!fBlockSti);
9551	Assert(!fBlockMovSS);
9552	}
9553	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
9554	{
9555	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
9556	Assert(!fBlockSti);
9557	Assert(!fBlockMovSS);
9558	Assert(!fBlockNmi);
9559	}
9560	#endif
9561	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9562	uIntType));
9563
9564	/*
9565	* Inject the event and get any changes to the guest-interruptibility state.
9566	*
9567	* The guest-interruptibility state may need to be updated if we inject the event
9568	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9569	*/
9570	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9571	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9572
9573	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9574	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9575	else
9576	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9577	}
9578
9579	/*
9580	* Update the guest-interruptibility state.
9581	*
9582	* This is required for the real-on-v86 software interrupt injection case above, as well as
9583	* updates to the guest state from ring-3 or IEM/REM.
9584	*/
9585	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9586	AssertRCReturn(rc, rc);
9587
9588	/*
9589	* There's no need to clear the VM-entry interruption-information field here if we're not
9590	* injecting anything. VT-x clears the valid bit on every VM-exit.
9591	*
9592	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9593	*/
9594
9595	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9596	NOREF(fBlockMovSS); NOREF(fBlockSti);
9597	return rcStrict;
9598	}
9599
9600
9601	/**
9602	* Enters the VT-x session.
9603	*
9604	* @returns VBox status code.
9605	* @param pVCpu The cross context virtual CPU structure.
9606	*/
9607	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9608	{
9609	AssertPtr(pVCpu);
9610	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9611	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9612
9613	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9614	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9615	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9616
9617	#ifdef VBOX_STRICT
9618	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9619	RTCCUINTREG uHostCr4 = ASMGetCR4();
9620	if (!(uHostCr4 & X86_CR4_VMXE))
9621	{
9622	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9623	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9624	}
9625	#endif
9626
9627	/*
9628	* Load the appropriate VMCS as the current and active one.
9629	*/
9630	PVMXVMCSINFO pVmcsInfo;
9631	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9632	if (!fInNestedGuestMode)
9633	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9634	else
9635	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9636	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9637	if (RT_SUCCESS(rc))
9638	{
9639	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9640	pVCpu->hm.s.fLeaveDone = false;
9641	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9642
9643	/*
9644	* Do the EMT scheduled L1D flush here if needed.
9645	*/
9646	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9647	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9648	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9649	hmR0MdsClear();
9650	}
9651	return rc;
9652	}
9653
9654
9655	/**
9656	* The thread-context callback (only on platforms which support it).
9657	*
9658	* @param enmEvent The thread-context event.
9659	* @param pVCpu The cross context virtual CPU structure.
9660	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9661	* @thread EMT(pVCpu)
9662	*/
9663	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9664	{
9665	AssertPtr(pVCpu);
9666	RT_NOREF1(fGlobalInit);
9667
9668	switch (enmEvent)
9669	{
9670	case RTTHREADCTXEVENT_OUT:
9671	{
9672	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9673	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9674	VMCPU_ASSERT_EMT(pVCpu);
9675
9676	/* No longjmps (logger flushes, locks) in this fragile context. */
9677	VMMRZCallRing3Disable(pVCpu);
9678	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9679
9680	/* Restore host-state (FPU, debug etc.) */
9681	if (!pVCpu->hm.s.fLeaveDone)
9682	{
9683	/*
9684	* Do -not- import the guest-state here as we might already be in the middle of importing
9685	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9686	*/
9687	hmR0VmxLeave(pVCpu, false /* fImportState */);
9688	pVCpu->hm.s.fLeaveDone = true;
9689	}
9690
9691	/* Leave HM context, takes care of local init (term). */
9692	int rc = HMR0LeaveCpu(pVCpu);
9693	AssertRC(rc);
9694
9695	/* Restore longjmp state. */
9696	VMMRZCallRing3Enable(pVCpu);
9697	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9698	break;
9699	}
9700
9701	case RTTHREADCTXEVENT_IN:
9702	{
9703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9704	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9705	VMCPU_ASSERT_EMT(pVCpu);
9706
9707	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9708	VMMRZCallRing3Disable(pVCpu);
9709	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9710
9711	/* Initialize the bare minimum state required for HM. This takes care of
9712	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9713	int rc = hmR0EnterCpu(pVCpu);
9714	AssertRC(rc);
9715	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9716	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9717
9718	/* Load the active VMCS as the current one. */
9719	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9720	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9721	AssertRC(rc);
9722	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9723	pVCpu->hm.s.fLeaveDone = false;
9724
9725	/* Do the EMT scheduled L1D flush if needed. */
9726	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9727	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9728
9729	/* Restore longjmp state. */
9730	VMMRZCallRing3Enable(pVCpu);
9731	break;
9732	}
9733
9734	default:
9735	break;
9736	}
9737	}
9738
9739
9740	/**
9741	* Exports the host state into the VMCS host-state area.
9742	* Sets up the VM-exit MSR-load area.
9743	*
9744	* The CPU state will be loaded from these fields on every successful VM-exit.
9745	*
9746	* @returns VBox status code.
9747	* @param pVCpu The cross context virtual CPU structure.
9748	*
9749	* @remarks No-long-jump zone!!!
9750	*/
9751	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9752	{
9753	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9754
9755	int rc = VINF_SUCCESS;
9756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9757	{
9758	rc = hmR0VmxExportHostControlRegs();
9759	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9760
9761	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9762	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9763
9764	rc = hmR0VmxExportHostMsrs(pVCpu);
9765	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9766
9767	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9768	}
9769	return rc;
9770	}
9771
9772
9773	/**
9774	* Saves the host state in the VMCS host-state.
9775	*
9776	* @returns VBox status code.
9777	* @param pVCpu The cross context virtual CPU structure.
9778	*
9779	* @remarks No-long-jump zone!!!
9780	*/
9781	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9782	{
9783	AssertPtr(pVCpu);
9784	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9785
9786	/*
9787	* Export the host state here while entering HM context.
9788	* When thread-context hooks are used, we might get preempted and have to re-save the host
9789	* state but most of the time we won't be, so do it here before we disable interrupts.
9790	*/
9791	return hmR0VmxExportHostState(pVCpu);
9792	}
9793
9794
9795	/**
9796	* Exports the guest state into the VMCS guest-state area.
9797	*
9798	* The will typically be done before VM-entry when the guest-CPU state and the
9799	* VMCS state may potentially be out of sync.
9800	*
9801	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9802	* VM-entry controls.
9803	* Sets up the appropriate VMX non-root function to execute guest code based on
9804	* the guest CPU mode.
9805	*
9806	* @returns VBox strict status code.
9807	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9808	* without unrestricted guest execution and the VMMDev is not presently
9809	* mapped (e.g. EFI32).
9810	*
9811	* @param pVCpu The cross context virtual CPU structure.
9812	* @param pVmxTransient The VMX-transient structure.
9813	*
9814	* @remarks No-long-jump zone!!!
9815	*/
9816	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9817	{
9818	AssertPtr(pVCpu);
9819	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9820	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9821
9822	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9823
9824	/*
9825	* Determine real-on-v86 mode.
9826	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9827	*/
9828	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9829	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9830	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9831	pVmcsInfo->RealMode. fRealOnV86Active = false;
9832	else
9833	{
9834	Assert(!pVmxTransient->fIsNestedGuest);
9835	pVmcsInfo->RealMode.fRealOnV86Active = true;
9836	}
9837
9838	/*
9839	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9840	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9841	*/
9842	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9843	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9844	* be a need to evaluate this everytime since I'm pretty sure we intercept
9845	* all guest paging mode changes. */
9846	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9847	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9848
9849	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9850	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9851
9852	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9853	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9854
9855	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9856	if (rcStrict == VINF_SUCCESS)
9857	{ /* likely */ }
9858	else
9859	{
9860	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9861	return rcStrict;
9862	}
9863
9864	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9865	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9866
9867	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9868	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9869
9870	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9871	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9872
9873	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9874	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9875
9876	rc = hmR0VmxExportGuestRip(pVCpu);
9877	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9878	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9879	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9880
9881	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9882	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9883
9884	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9885	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9886	\| HM_CHANGED_GUEST_CR2
9887	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9888	\| HM_CHANGED_GUEST_X87
9889	\| HM_CHANGED_GUEST_SSE_AVX
9890	\| HM_CHANGED_GUEST_OTHER_XSAVE
9891	\| HM_CHANGED_GUEST_XCRx
9892	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9893	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9894	\| HM_CHANGED_GUEST_TSC_AUX
9895	\| HM_CHANGED_GUEST_OTHER_MSRS
9896	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9897
9898	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9899	return rc;
9900	}
9901
9902
9903	/**
9904	* Exports the state shared between the host and guest into the VMCS.
9905	*
9906	* @param pVCpu The cross context virtual CPU structure.
9907	* @param pVmxTransient The VMX-transient structure.
9908	*
9909	* @remarks No-long-jump zone!!!
9910	*/
9911	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9912	{
9913	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9914	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9915
9916	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9917	{
9918	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9919	AssertRC(rc);
9920	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9921
9922	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9923	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9924	{
9925	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9926	AssertRC(rc);
9927	}
9928	}
9929
9930	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9931	{
9932	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9933	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9934	}
9935
9936	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9937	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9938	}
9939
9940
9941	/**
9942	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9943	*
9944	* @returns Strict VBox status code (i.e. informational status codes too).
9945	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9946	* without unrestricted guest execution and the VMMDev is not presently
9947	* mapped (e.g. EFI32).
9948	*
9949	* @param pVCpu The cross context virtual CPU structure.
9950	* @param pVmxTransient The VMX-transient structure.
9951	*
9952	* @remarks No-long-jump zone!!!
9953	*/
9954	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9955	{
9956	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9957	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9958	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9959
9960	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9961	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9962	#endif
9963
9964	/*
9965	* For many exits it's only RIP that changes and hence try to export it first
9966	* without going through a lot of change flag checks.
9967	*/
9968	VBOXSTRICTRC rcStrict;
9969	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9970	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9971	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9972	{
9973	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9974	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9975	{ /* likely */}
9976	else
9977	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9978	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9979	}
9980	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9981	{
9982	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9983	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9984	{ /* likely */}
9985	else
9986	{
9987	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9988	VBOXSTRICTRC_VAL(rcStrict)));
9989	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9990	return rcStrict;
9991	}
9992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9993	}
9994	else
9995	rcStrict = VINF_SUCCESS;
9996
9997	#ifdef VBOX_STRICT
9998	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9999	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
10000	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
10001	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
10002	("fCtxChanged=%#RX64\n", fCtxChanged));
10003	#endif
10004	return rcStrict;
10005	}
10006
10007
10008	/**
10009	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10010	* and update error record fields accordingly.
10011	*
10012	* @returns VMX_IGS_* error codes.
10013	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10014	* wrong with the guest state.
10015	*
10016	* @param pVCpu The cross context virtual CPU structure.
10017	* @param pVmcsInfo The VMCS info. object.
10018	*
10019	* @remarks This function assumes our cache of the VMCS controls
10020	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10021	*/
10022	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10023	{
10024	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10025	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10026	uError = (err); \
10027	break; \
10028	} else do { } while (0)
10029
10030	int rc;
10031	PVM pVM = pVCpu->CTX_SUFF(pVM);
10032	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10033	uint32_t uError = VMX_IGS_ERROR;
10034	uint32_t u32Val;
10035	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10036
10037	do
10038	{
10039	/*
10040	* CR0.
10041	*/
10042	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10043	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10044	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10045	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
10046	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10047	if (fUnrestrictedGuest)
10048	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10049
10050	uint32_t u32GuestCr0;
10051	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10052	AssertRCBreak(rc);
10053	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10054	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10055	if ( !fUnrestrictedGuest
10056	&& (u32GuestCr0 & X86_CR0_PG)
10057	&& !(u32GuestCr0 & X86_CR0_PE))
10058	{
10059	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10060	}
10061
10062	/*
10063	* CR4.
10064	*/
10065	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
10066	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10067	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10068
10069	uint32_t u32GuestCr4;
10070	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10071	AssertRCBreak(rc);
10072	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10073	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10074
10075	/*
10076	* IA32_DEBUGCTL MSR.
10077	*/
10078	uint64_t u64Val;
10079	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10080	AssertRCBreak(rc);
10081	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10082	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10083	{
10084	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10085	}
10086	uint64_t u64DebugCtlMsr = u64Val;
10087
10088	#ifdef VBOX_STRICT
10089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10090	AssertRCBreak(rc);
10091	Assert(u32Val == pVmcsInfo->u32EntryCtls);
10092	#endif
10093	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10094
10095	/*
10096	* RIP and RFLAGS.
10097	*/
10098	uint32_t u32Eflags;
10099	#if HC_ARCH_BITS == 64
10100	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10101	AssertRCBreak(rc);
10102	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10103	if ( !fLongModeGuest
10104	\|\| !pCtx->cs.Attr.n.u1Long)
10105	{
10106	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10107	}
10108	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10109	* must be identical if the "IA-32e mode guest" VM-entry
10110	* control is 1 and CS.L is 1. No check applies if the
10111	* CPU supports 64 linear-address bits. */
10112
10113	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10114	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10115	AssertRCBreak(rc);
10116	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10117	VMX_IGS_RFLAGS_RESERVED);
10118	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10119	u32Eflags = u64Val;
10120	#else
10121	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10122	AssertRCBreak(rc);
10123	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10124	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10125	#endif
10126
10127	if ( fLongModeGuest
10128	\|\| ( fUnrestrictedGuest
10129	&& !(u32GuestCr0 & X86_CR0_PE)))
10130	{
10131	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10132	}
10133
10134	uint32_t u32EntryInfo;
10135	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10136	AssertRCBreak(rc);
10137	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10138	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10139	{
10140	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10141	}
10142
10143	/*
10144	* 64-bit checks.
10145	*/
10146	#if HC_ARCH_BITS == 64
10147	if (fLongModeGuest)
10148	{
10149	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10150	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10151	}
10152
10153	if ( !fLongModeGuest
10154	&& (u32GuestCr4 & X86_CR4_PCIDE))
10155	{
10156	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10157	}
10158
10159	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10160	* 51:32 beyond the processor's physical-address width are 0. */
10161
10162	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10163	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10164	{
10165	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10166	}
10167
10168	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10169	AssertRCBreak(rc);
10170	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10171
10172	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10173	AssertRCBreak(rc);
10174	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10175	#endif
10176
10177	/*
10178	* PERF_GLOBAL MSR.
10179	*/
10180	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10181	{
10182	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10183	AssertRCBreak(rc);
10184	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10185	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10186	}
10187
10188	/*
10189	* PAT MSR.
10190	*/
10191	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10192	{
10193	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10194	AssertRCBreak(rc);
10195	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10196	for (unsigned i = 0; i < 8; i++)
10197	{
10198	uint8_t u8Val = (u64Val & 0xff);
10199	if ( u8Val != 0 /* UC */
10200	&& u8Val != 1 /* WC */
10201	&& u8Val != 4 /* WT */
10202	&& u8Val != 5 /* WP */
10203	&& u8Val != 6 /* WB */
10204	&& u8Val != 7 /* UC- */)
10205	{
10206	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10207	}
10208	u64Val >>= 8;
10209	}
10210	}
10211
10212	/*
10213	* EFER MSR.
10214	*/
10215	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10216	{
10217	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10218	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10219	AssertRCBreak(rc);
10220	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10221	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10222	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
10223	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10224	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10225	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10226	* iemVmxVmentryCheckGuestState(). */
10227	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10228	\|\| !(u32GuestCr0 & X86_CR0_PG)
10229	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10230	VMX_IGS_EFER_LMA_LME_MISMATCH);
10231	}
10232
10233	/*
10234	* Segment registers.
10235	*/
10236	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10237	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10238	if (!(u32Eflags & X86_EFL_VM))
10239	{
10240	/* CS */
10241	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10242	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10243	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10244	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10245	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10246	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10247	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10248	/* CS cannot be loaded with NULL in protected mode. */
10249	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10250	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10251	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10252	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10253	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10254	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10255	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10256	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10257	else
10258	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10259
10260	/* SS */
10261	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10262	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10263	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10264	if ( !(pCtx->cr0 & X86_CR0_PE)
10265	\|\| pCtx->cs.Attr.n.u4Type == 3)
10266	{
10267	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10268	}
10269	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10270	{
10271	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10272	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10273	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10274	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10275	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10276	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10277	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10278	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10279	}
10280
10281	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
10282	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10283	{
10284	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10285	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10286	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10287	\|\| pCtx->ds.Attr.n.u4Type > 11
10288	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10289	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10290	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10291	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10292	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10293	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10294	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10295	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10296	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10297	}
10298	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10299	{
10300	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10301	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10302	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10303	\|\| pCtx->es.Attr.n.u4Type > 11
10304	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10305	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10306	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10307	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10308	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10309	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10310	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10311	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10312	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10313	}
10314	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10315	{
10316	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10317	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10318	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10319	\|\| pCtx->fs.Attr.n.u4Type > 11
10320	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10321	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10322	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10323	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10324	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10325	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10326	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10327	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10328	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10329	}
10330	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10331	{
10332	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10333	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10334	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10335	\|\| pCtx->gs.Attr.n.u4Type > 11
10336	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10337	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10338	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10339	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10340	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10341	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10342	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10343	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10344	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10345	}
10346	/* 64-bit capable CPUs. */
10347	#if HC_ARCH_BITS == 64
10348	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10349	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10350	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10351	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10352	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10353	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10354	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10355	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10356	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10357	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10358	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10359	#endif
10360	}
10361	else
10362	{
10363	/* V86 mode checks. */
10364	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10365	if (pVmcsInfo->RealMode.fRealOnV86Active)
10366	{
10367	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10368	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10369	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10370	}
10371	else
10372	{
10373	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10374	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10375	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10376	}
10377
10378	/* CS */
10379	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10380	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10381	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10382	/* SS */
10383	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10384	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10385	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10386	/* DS */
10387	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10388	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10389	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10390	/* ES */
10391	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10392	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10393	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10394	/* FS */
10395	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10396	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10397	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10398	/* GS */
10399	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10400	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10401	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10402	/* 64-bit capable CPUs. */
10403	#if HC_ARCH_BITS == 64
10404	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10405	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10406	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10407	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10408	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10409	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
10410	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10411	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
10412	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10413	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
10414	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10415	#endif
10416	}
10417
10418	/*
10419	* TR.
10420	*/
10421	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10422	/* 64-bit capable CPUs. */
10423	#if HC_ARCH_BITS == 64
10424	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10425	#endif
10426	if (fLongModeGuest)
10427	{
10428	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10429	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10430	}
10431	else
10432	{
10433	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10434	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10435	VMX_IGS_TR_ATTR_TYPE_INVALID);
10436	}
10437	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10438	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10439	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10440	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10441	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10442	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10443	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10444	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10445
10446	/*
10447	* GDTR and IDTR.
10448	*/
10449	#if HC_ARCH_BITS == 64
10450	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10451	AssertRCBreak(rc);
10452	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10453
10454	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10455	AssertRCBreak(rc);
10456	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10457	#endif
10458
10459	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10460	AssertRCBreak(rc);
10461	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10462
10463	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10464	AssertRCBreak(rc);
10465	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10466
10467	/*
10468	* Guest Non-Register State.
10469	*/
10470	/* Activity State. */
10471	uint32_t u32ActivityState;
10472	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10473	AssertRCBreak(rc);
10474	HMVMX_CHECK_BREAK( !u32ActivityState
10475	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10476	VMX_IGS_ACTIVITY_STATE_INVALID);
10477	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10478	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10479	uint32_t u32IntrState;
10480	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10481	AssertRCBreak(rc);
10482	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10483	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10484	{
10485	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10486	}
10487
10488	/** @todo Activity state and injecting interrupts. Left as a todo since we
10489	* currently don't use activity states but ACTIVE. */
10490
10491	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10492	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10493
10494	/* Guest interruptibility-state. */
10495	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10496	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10497	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10498	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10499	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10500	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10501	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10502	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10503	{
10504	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10505	{
10506	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10507	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10508	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10509	}
10510	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10511	{
10512	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10513	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10514	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10515	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10516	}
10517	}
10518	/** @todo Assumes the processor is not in SMM. */
10519	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10520	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10521	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10522	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10523	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10524	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10525	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10526	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10527	{
10528	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10529	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10530	}
10531
10532	/* Pending debug exceptions. */
10533	#if HC_ARCH_BITS == 64
10534	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10535	AssertRCBreak(rc);
10536	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10537	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10538	u32Val = u64Val; /* For pending debug exceptions checks below. */
10539	#else
10540	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10541	AssertRCBreak(rc);
10542	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10543	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10544	#endif
10545
10546	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10547	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10548	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10549	{
10550	if ( (u32Eflags & X86_EFL_TF)
10551	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10552	{
10553	/* Bit 14 is PendingDebug.BS. */
10554	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10555	}
10556	if ( !(u32Eflags & X86_EFL_TF)
10557	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10558	{
10559	/* Bit 14 is PendingDebug.BS. */
10560	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10561	}
10562	}
10563
10564	/* VMCS link pointer. */
10565	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10566	AssertRCBreak(rc);
10567	if (u64Val != UINT64_C(0xffffffffffffffff))
10568	{
10569	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10570	/** @todo Bits beyond the processor's physical-address width MBZ. */
10571	/** @todo SMM checks. */
10572	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10573	Assert(pVmcsInfo->pvShadowVmcs);
10574	VMXVMCSREVID VmcsRevId;
10575	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10576	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10577	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10578	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10579	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10580	}
10581
10582	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10583	* not using nested paging? */
10584	if ( pVM->hm.s.fNestedPaging
10585	&& !fLongModeGuest
10586	&& CPUMIsGuestInPAEModeEx(pCtx))
10587	{
10588	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10589	AssertRCBreak(rc);
10590	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10591
10592	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10593	AssertRCBreak(rc);
10594	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10595
10596	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10597	AssertRCBreak(rc);
10598	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10599
10600	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10601	AssertRCBreak(rc);
10602	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10603	}
10604
10605	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10606	if (uError == VMX_IGS_ERROR)
10607	uError = VMX_IGS_REASON_NOT_FOUND;
10608	} while (0);
10609
10610	pVCpu->hm.s.u32HMError = uError;
10611	return uError;
10612
10613	#undef HMVMX_ERROR_BREAK
10614	#undef HMVMX_CHECK_BREAK
10615	}
10616
10617
10618	/**
10619	* Map the APIC-access page for virtualizing APIC accesses.
10620	*
10621	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10622	* this not done as part of exporting guest state, see @bugref{8721}.
10623	*
10624	* @returns VBox status code.
10625	* @param pVCpu The cross context virtual CPU structure.
10626	*/
10627	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10628	{
10629	PVM pVM = pVCpu->CTX_SUFF(pVM);
10630	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10631
10632	Assert(PDMHasApic(pVM));
10633	Assert(u64MsrApicBase);
10634
10635	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10636	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10637
10638	/* Unalias the existing mapping. */
10639	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10640	AssertRCReturn(rc, rc);
10641
10642	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10643	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10644	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10645	AssertRCReturn(rc, rc);
10646
10647	/* Update the per-VCPU cache of the APIC base MSR. */
10648	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10649	return VINF_SUCCESS;
10650	}
10651
10652
10653	/**
10654	* Wrapper for dispatching host NMIs.
10655	*
10656	* @returns VBox status code.
10657	* @param pVCpu The cross context virtual CPU structure.
10658	*/
10659	static int hmR0VmxExitHostNmi(PVMCPU pVCpu)
10660	{
10661	VMXDispatchHostNmi();
10662	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10663	return VINF_SUCCESS;
10664	}
10665
10666
10667	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10668	/**
10669	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10670	* nested-guest using hardware-assisted VMX.
10671	*
10672	* @param pVCpu The cross context virtual CPU structure.
10673	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10674	* @param pVmcsInfoGst The guest VMCS info. object.
10675	*/
10676	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10677	{
10678	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10679	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10680	Assert(pu64MsrBitmap);
10681
10682	/*
10683	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10684	* MSR that is intercepted by the guest is also intercepted while executing the
10685	* nested-guest using hardware-assisted VMX.
10686	*
10687	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10688	* nested-guest VM-exit even if the outer guest is not intercepting some
10689	* MSRs. We cannot assume the caller has initialized the nested-guest
10690	* MSR bitmap in this case.
10691	*
10692	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10693	* each VM-entry, hence initializing it once per-VM while setting up the
10694	* nested-guest VMCS is not sufficient.
10695	*/
10696	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10697	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10698	{
10699	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10700	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10701	Assert(pu64MsrBitmapNstGst);
10702	Assert(pu64MsrBitmapGst);
10703
10704	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10705	for (uint32_t i = 0; i < cFrags; i++)
10706	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10707	}
10708	else
10709	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10710	}
10711
10712
10713	/**
10714	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10715	* hardware-assisted VMX execution of the nested-guest.
10716	*
10717	* For a guest, we don't modify these controls once we set up the VMCS and hence
10718	* this function is never called.
10719	*
10720	* For nested-guests since the guest hypervisor provides these controls on every
10721	* nested-guest VM-entry and could potentially change them everytime we need to
10722	* merge them before every nested-guest VM-entry.
10723	*
10724	* @returns VBox status code.
10725	* @param pVCpu The cross context virtual CPU structure.
10726	*/
10727	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10728	{
10729	PVM pVM = pVCpu->CTX_SUFF(pVM);
10730	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10731	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10732	Assert(pVmcsNstGst);
10733
10734	/*
10735	* Merge the controls with the requirements of the guest VMCS.
10736	*
10737	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10738	* VMCS with the features supported by the physical CPU as it's already done by the
10739	* VMLAUNCH/VMRESUME instruction emulation.
10740	*
10741	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10742	* derived from the VMX features supported by the physical CPU.
10743	*/
10744
10745	/* Pin-based VM-execution controls. */
10746	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10747
10748	/* Processor-based VM-execution controls. */
10749	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10750	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10751	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10752	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10753	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10754
10755	/* Secondary processor-based VM-execution controls. */
10756	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10757	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10758	\| VMX_PROC_CTLS2_INVPCID
10759	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10760	\| VMX_PROC_CTLS2_RDTSCP
10761	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10762	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10763	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10764	\| VMX_PROC_CTLS2_VMFUNC));
10765
10766	/*
10767	* VM-entry controls:
10768	* These controls contains state that depends on the nested-guest state (primarily
10769	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10770	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10771	* properly continue executing the nested-guest if the EFER MSR changes but does not
10772	* cause a nested-guest VM-exits.
10773	*
10774	* VM-exit controls:
10775	* These controls specify the host state on return. We cannot use the controls from
10776	* the guest hypervisor state as is as it would contain the guest state rather than
10777	* the host state. Since the host state is subject to change (e.g. preemption, trips
10778	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10779	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10780	*
10781	* VM-entry MSR-load:
10782	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10783	* context by the VMLAUNCH/VMRESUME instruction emulation.
10784	*
10785	* VM-exit MSR-store:
10786	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10787	* back into the VM-exit MSR-store area.
10788	*
10789	* VM-exit MSR-load areas:
10790	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10791	* can entirely ignore what the guest hypervisor wants to load here.
10792	*/
10793
10794	/*
10795	* Exception bitmap.
10796	*
10797	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10798	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10799	* code more flexible if intercepting exceptions become more dynamic in the future we do
10800	* it as part of exporting the nested-guest state.
10801	*/
10802	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10803
10804	/*
10805	* CR0/CR4 guest/host mask.
10806	*
10807	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10808	* cause VM-exits, so we need to merge them here.
10809	*/
10810	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10811	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10812
10813	/*
10814	* Page-fault error-code mask and match.
10815	*
10816	* Although we require unrestricted guest execution (and thereby nested-paging) for
10817	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10818	* normally intercept #PFs, it might intercept them for debugging purposes.
10819	*
10820	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10821	* If the outer guest is intercepting #PFs we must intercept all #PFs.
10822	*/
10823	uint32_t u32XcptPFMask;
10824	uint32_t u32XcptPFMatch;
10825	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10826	{
10827	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10828	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10829	}
10830	else
10831	{
10832	u32XcptPFMask = 0;
10833	u32XcptPFMatch = 0;
10834	}
10835
10836	/*
10837	* Pause-Loop exiting.
10838	*/
10839	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10840	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10841
10842	/*
10843	* I/O Bitmap.
10844	*
10845	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10846	* intercept all I/O port accesses.
10847	*/
10848	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10849
10850	/*
10851	* VMCS shadowing.
10852	*
10853	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10854	* enabled while executing the nested-guest.
10855	*/
10856	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10857
10858	/*
10859	* APIC-access page.
10860	*
10861	* The APIC-access page address has already been initialized while setting up the
10862	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it should
10863	* not be of any consequence to the host or to the guest for that matter, but we only
10864	* accept valid addresses verified by the VMLAUNCH/VMRESUME instruction emulation to keep
10865	* it simple.
10866	*/
10867
10868	/*
10869	* Virtual-APIC page and TPR threshold.
10870	*
10871	* The virtual-APIC page has already been allocated (by CPUM during VM startup) and cached
10872	* from guest memory as part of VMLAUNCH/VMRESUME instruction emulation. The host physical
10873	* address has also been updated in the nested-guest VMCS.
10874	*/
10875	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10876	RTHCPHYS HCPhysVirtApic;
10877	uint32_t u32TprThreshold;
10878	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10879	{
10880	HCPhysVirtApic = pVmcsInfoNstGst->HCPhysVirtApic;
10881	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10882	}
10883	else
10884	{
10885	HCPhysVirtApic = 0;
10886	u32TprThreshold = 0;
10887
10888	/*
10889	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10890	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10891	* be taken care of by EPT/shadow paging.
10892	*/
10893	if (pVM->hm.s.fAllow64BitGuests)
10894	{
10895	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10896	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10897	}
10898	}
10899
10900	/*
10901	* Validate basic assumptions.
10902	*/
10903	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10904	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10905	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10906
10907	/*
10908	* Commit it to the nested-guest VMCS.
10909	*/
10910	int rc = VINF_SUCCESS;
10911	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10912	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10913	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10914	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10915	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10916	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10917	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10918	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10919	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10920	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10921	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10922	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10923	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10924	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10925	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10926	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10927	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10928	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10929	{
10930	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10931	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10932	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10933	}
10934	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10935	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVmcsInfoNstGst->HCPhysVirtApic);
10936	AssertRCReturn(rc, rc);
10937
10938	/*
10939	* Update the nested-guest VMCS cache.
10940	*/
10941	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10942	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10943	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10944	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10945	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10946	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10947	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10948	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10949	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10950
10951	/*
10952	* MSR bitmap.
10953	*
10954	* The MSR bitmap address has already been initialized while setting up the nested-guest
10955	* VMCS, here we need to merge the MSR bitmaps.
10956	*/
10957	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10958	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10959
10960	return VINF_SUCCESS;
10961	}
10962	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10963
10964
10965	/**
10966	* Does the preparations before executing guest code in VT-x.
10967	*
10968	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10969	* recompiler/IEM. We must be cautious what we do here regarding committing
10970	* guest-state information into the VMCS assuming we assuredly execute the
10971	* guest in VT-x mode.
10972	*
10973	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10974	* the common-state (TRPM/forceflags), we must undo those changes so that the
10975	* recompiler/IEM can (and should) use them when it resumes guest execution.
10976	* Otherwise such operations must be done when we can no longer exit to ring-3.
10977	*
10978	* @returns Strict VBox status code (i.e. informational status codes too).
10979	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10980	* have been disabled.
10981	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10982	* pending events).
10983	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10984	* double-fault into the guest.
10985	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10986	* dispatched directly.
10987	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10988	*
10989	* @param pVCpu The cross context virtual CPU structure.
10990	* @param pVmxTransient The VMX-transient structure.
10991	* @param fStepping Whether we are single-stepping the guest in the
10992	* hypervisor debugger. Makes us ignore some of the reasons
10993	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10994	* if event dispatching took place.
10995	*/
10996	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10997	{
10998	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10999
11000	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
11001	if (pVmxTransient->fIsNestedGuest)
11002	{
11003	RT_NOREF2(pVCpu, fStepping);
11004	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
11005	return VINF_EM_RESCHEDULE_REM;
11006	}
11007	#endif
11008
11009	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
11010	PGMRZDynMapFlushAutoSet(pVCpu);
11011	#endif
11012
11013	/*
11014	* Check and process force flag actions, some of which might require us to go back to ring-3.
11015	*/
11016	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
11017	if (rcStrict == VINF_SUCCESS)
11018	{ /* FFs don't get set all the time. */ }
11019	else
11020	return rcStrict;
11021
11022	/*
11023	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
11024	*/
11025	/** @todo Doing this from ring-3 after VM setup phase causes a
11026	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
11027	* idea why atm. */
11028	PVM pVM = pVCpu->CTX_SUFF(pVM);
11029	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
11030	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
11031	&& PDMHasApic(pVM))
11032	{
11033	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
11034	AssertRCReturn(rc, rc);
11035	}
11036
11037	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11038	/*
11039	* Merge guest VMCS controls with the nested-guest VMCS controls.
11040	*
11041	* Even if we have not executed the guest prior to this (e.g. when resuming from a
11042	* saved state), we should be okay with merging controls as we initialize the
11043	* guest VMCS controls as part of VM setup phase.
11044	*/
11045	if ( pVmxTransient->fIsNestedGuest
11046	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
11047	{
11048	int rc = hmR0VmxMergeVmcsNested(pVCpu);
11049	AssertRCReturn(rc, rc);
11050	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
11051	}
11052	#endif
11053
11054	/*
11055	* Evaluate events to be injected into the guest.
11056	*
11057	* Events in TRPM can be injected without inspecting the guest state.
11058	* If any new events (interrupts/NMI) are pending currently, we try to set up the
11059	* guest to cause a VM-exit the next time they are ready to receive the event.
11060	*
11061	* With nested-guests, evaluating pending events may cause VM-exits.
11062	*/
11063	if (TRPMHasTrap(pVCpu))
11064	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
11065
11066	uint32_t fIntrState;
11067	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
11068
11069	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11070	/*
11071	* While evaluating pending events if something failed (unlikely) or if we were
11072	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
11073	*/
11074	if (rcStrict != VINF_SUCCESS)
11075	return rcStrict;
11076	if ( pVmxTransient->fIsNestedGuest
11077	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11078	{
11079	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11080	return VINF_VMX_VMEXIT;
11081	}
11082	#else
11083	Assert(rcStrict == VINF_SUCCESS);
11084	#endif
11085
11086	/*
11087	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
11088	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
11089	* also result in triple-faulting the VM.
11090	*
11091	* With nested-guests, the above does not apply since unrestricted guest execution is a
11092	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
11093	*/
11094	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
11095	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11096	{ /* likely */ }
11097	else
11098	{
11099	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
11100	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11101	return rcStrict;
11102	}
11103
11104	/*
11105	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
11106	* import CR3 themselves. We will need to update them here, as even as late as the above
11107	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
11108	* the below force flags to be set.
11109	*/
11110	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
11111	{
11112	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
11113	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
11114	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
11115	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
11116	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11117	}
11118	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
11119	{
11120	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
11121	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11122	}
11123
11124	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11125	/* Paranoia. */
11126	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11127	#endif
11128
11129	/*
11130	* No longjmps to ring-3 from this point on!!!
11131	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
11132	* This also disables flushing of the R0-logger instance (if any).
11133	*/
11134	VMMRZCallRing3Disable(pVCpu);
11135
11136	/*
11137	* Export the guest state bits.
11138	*
11139	* We cannot perform longjmps while loading the guest state because we do not preserve the
11140	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
11141	* CPU migration.
11142	*
11143	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
11144	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
11145	*/
11146	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
11147	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11148	{ /* likely */ }
11149	else
11150	{
11151	VMMRZCallRing3Enable(pVCpu);
11152	return rcStrict;
11153	}
11154
11155	/*
11156	* We disable interrupts so that we don't miss any interrupts that would flag preemption
11157	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
11158	* preemption disabled for a while. Since this is purely to aid the
11159	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
11160	* disable interrupt on NT.
11161	*
11162	* We need to check for force-flags that could've possible been altered since we last
11163	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
11164	* see @bugref{6398}).
11165	*
11166	* We also check a couple of other force-flags as a last opportunity to get the EMT back
11167	* to ring-3 before executing guest code.
11168	*/
11169	pVmxTransient->fEFlags = ASMIntDisableFlags();
11170
11171	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
11172	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
11173	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
11174	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
11175	{
11176	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
11177	{
11178	pVCpu->hm.s.Event.fPending = false;
11179
11180	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11181	/*
11182	* If we are executing a nested-guest make sure that we should intercept subsequent
11183	* events. The one we are injecting might be part of VM-entry.
11184	*/
11185	if (pVmxTransient->fIsNestedGuest)
11186	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
11187	#endif
11188
11189	/*
11190	* We've injected any pending events. This is really the point of no return (to ring-3).
11191	*
11192	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
11193	* returns from this function, so don't enable them here.
11194	*/
11195	return VINF_SUCCESS;
11196	}
11197
11198	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
11199	rcStrict = VINF_EM_RAW_INTERRUPT;
11200	}
11201	else
11202	{
11203	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
11204	rcStrict = VINF_EM_RAW_TO_R3;
11205	}
11206
11207	ASMSetFlags(pVmxTransient->fEFlags);
11208	VMMRZCallRing3Enable(pVCpu);
11209
11210	return rcStrict;
11211	}
11212
11213
11214	/**
11215	* Final preparations before executing guest code using hardware-assisted VMX.
11216	*
11217	* We can no longer get preempted to a different host CPU and there are no returns
11218	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
11219	* failures), this function is not intended to fail sans unrecoverable hardware
11220	* errors.
11221	*
11222	* @param pVCpu The cross context virtual CPU structure.
11223	* @param pVmxTransient The VMX-transient structure.
11224	*
11225	* @remarks Called with preemption disabled.
11226	* @remarks No-long-jump zone!!!
11227	*/
11228	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11229	{
11230	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11231	Assert(VMMR0IsLogFlushDisabled(pVCpu));
11232	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
11233	Assert(!pVCpu->hm.s.Event.fPending);
11234
11235	/*
11236	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
11237	*/
11238	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11239	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
11240
11241	PVM pVM = pVCpu->CTX_SUFF(pVM);
11242	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11243	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
11244	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
11245
11246	if (!CPUMIsGuestFPUStateActive(pVCpu))
11247	{
11248	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11249	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
11250	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
11251	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
11252	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
11253	}
11254
11255	/*
11256	* Re-export the host state bits as we may've been preempted (only happens when
11257	* thread-context hooks are used or when the VM start function changes) or if
11258	* the host CR0 is modified while loading the guest FPU state above.
11259	*
11260	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
11261	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
11262	* see @bugref{8432}.
11263	*
11264	* This may also happen when switching to/from a nested-guest VMCS without leaving
11265	* ring-0.
11266	*/
11267	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
11268	{
11269	hmR0VmxExportHostState(pVCpu);
11270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
11271	}
11272	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
11273
11274	/*
11275	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
11276	*/
11277	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
11278	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
11279	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
11280
11281	/*
11282	* Store status of the shared guest/host debug state at the time of VM-entry.
11283	*/
11284	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
11285	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
11286	{
11287	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
11288	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
11289	}
11290	else
11291	#endif
11292	{
11293	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
11294	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
11295	}
11296
11297	/*
11298	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
11299	* more than one conditional check. The post-run side of our code shall determine
11300	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
11301	*/
11302	if (pVmcsInfo->pbVirtApic)
11303	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
11304
11305	/*
11306	* Update the host MSRs values in the VM-exit MSR-load area.
11307	*/
11308	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
11309	{
11310	if (pVmcsInfo->cExitMsrLoad > 0)
11311	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
11312	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
11313	}
11314
11315	/*
11316	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
11317	* VMX-preemption timer based on the next virtual sync clock deadline.
11318	*/
11319	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
11320	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
11321	{
11322	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
11323	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
11324	}
11325
11326	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
11327	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
11328	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
11329	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
11330	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
11331
11332	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
11333
11334	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
11335	as we're about to start executing the guest . */
11336
11337	/*
11338	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
11339	*
11340	* This is done this late as updating the TSC offsetting/preemption timer above
11341	* figures out if we can skip intercepting RDTSCP by calculating the number of
11342	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
11343	*/
11344	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
11345	&& !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11346	{
11347	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
11348
11349	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
11350	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
11351	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
11352	true /* fSetReadWrite /, true / fUpdateHostMsr */);
11353	AssertRC(rc);
11354	Assert(!pVmxTransient->fRemoveTscAuxMsr);
11355	pVmxTransient->fRemoveTscAuxMsr = true;
11356	}
11357
11358	#ifdef VBOX_STRICT
11359	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
11360	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
11361	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
11362	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
11363	#endif
11364
11365	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
11366	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
11367	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
11368	* see @bugref{9180#c54}. */
11369	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
11370	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
11371	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
11372	#endif
11373	}
11374
11375
11376	/**
11377	* First C routine invoked after running guest code using hardware-assisted VMX.
11378	*
11379	* @param pVCpu The cross context virtual CPU structure.
11380	* @param pVmxTransient The VMX-transient structure.
11381	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
11382	*
11383	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
11384	*
11385	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
11386	* unconditionally when it is safe to do so.
11387	*/
11388	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
11389	{
11390	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
11391
11392	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
11393	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
11394	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
11395	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
11396	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
11397	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
11398
11399	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11400	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
11401	{
11402	uint64_t uGstTsc;
11403	if (!pVmxTransient->fIsNestedGuest)
11404	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11405	else
11406	{
11407	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
11408	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
11409	}
11410	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
11411	}
11412
11413	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
11414	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
11415	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
11416
11417	#if HC_ARCH_BITS == 64
11418	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
11419	#endif
11420	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
11421	/* The 64-on-32 switcher maintains VMCS-launch state on its own
11422	and we need to leave it alone here. */
11423	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
11424	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11425	#else
11426	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
11427	#endif
11428	#ifdef VBOX_STRICT
11429	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
11430	#endif
11431	Assert(!ASMIntAreEnabled());
11432	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
11433	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
11434
11435	/*
11436	* Save the basic VM-exit reason and check if the VM-entry failed.
11437	* See Intel spec. 24.9.1 "Basic VM-exit Information".
11438	*/
11439	uint32_t uExitReason;
11440	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11441	AssertRC(rc);
11442	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11443	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11444
11445	/*
11446	* Log the VM-exit before logging anything else as otherwise it might be a
11447	* tad confusing what happens before and after the world-switch.
11448	*/
11449	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11450
11451	/*
11452	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11453	* bitmap permissions, if it was added before VM-entry.
11454	*/
11455	if (pVmxTransient->fRemoveTscAuxMsr)
11456	{
11457	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11458	pVmxTransient->fRemoveTscAuxMsr = false;
11459	}
11460
11461	/*
11462	* Check if VMLAUNCH/VMRESUME succeeded.
11463	* If this failed, we cause a guru meditation and cease further execution.
11464	*
11465	* However, if we are executing a nested-guest we might fail if we use the
11466	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11467	*/
11468	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11469	{
11470	/*
11471	* Update the VM-exit history array here even if the VM-entry failed due to:
11472	* - Invalid guest state.
11473	* - MSR loading.
11474	* - Machine-check event.
11475	*
11476	* In any of the above cases we will still have a "valid" VM-exit reason
11477	* despite @a fVMEntryFailed being false.
11478	*
11479	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11480	*
11481	* Note! We don't have CS or RIP at this point. Will probably address that later
11482	* by amending the history entry added here.
11483	*/
11484	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11485	UINT64_MAX, uHostTsc);
11486
11487	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11488	{
11489	VMMRZCallRing3Enable(pVCpu);
11490
11491	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11492	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11493
11494	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11495	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11496	AssertRC(rc);
11497	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11498	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11499	AssertRC(rc);
11500	#else
11501	/*
11502	* Import the guest-interruptibility state always as we need it while evaluating
11503	* injecting events on re-entry.
11504	*
11505	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11506	* checking for real-mode while exporting the state because all bits that cause
11507	* mode changes wrt CR0 are intercepted.
11508	*/
11509	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11510	AssertRC(rc);
11511	#endif
11512
11513	/*
11514	* Sync the TPR shadow with our APIC state.
11515	*
11516	* With nested-guests, mark the virtual-APIC page as dirty so it can be synced
11517	* when performing the nested-guest VM-exit.
11518	*/
11519	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
11520	{
11521	if (!pVmxTransient->fIsNestedGuest)
11522	{
11523	Assert(pVmcsInfo->pbVirtApic);
11524	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11525	{
11526	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11527	AssertRC(rc);
11528	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11529	}
11530	}
11531	else
11532	pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtApicPageDirty = true;
11533	}
11534
11535	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11536	return;
11537	}
11538	}
11539	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11540	else if (pVmxTransient->fIsNestedGuest)
11541	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11542	#endif
11543	else
11544	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11545
11546	VMMRZCallRing3Enable(pVCpu);
11547	}
11548
11549
11550	/**
11551	* Runs the guest code using hardware-assisted VMX the normal way.
11552	*
11553	* @returns VBox status code.
11554	* @param pVCpu The cross context virtual CPU structure.
11555	* @param pcLoops Pointer to the number of executed loops.
11556	*/
11557	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11558	{
11559	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11560	Assert(pcLoops);
11561	Assert(*pcLoops <= cMaxResumeLoops);
11562	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11563
11564	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11565	/*
11566	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11567	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11568	* guest VMCS while entering the VMX ring-0 session.
11569	*/
11570	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11571	{
11572	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11573	if (RT_SUCCESS(rc))
11574	{ /* likely */ }
11575	else
11576	{
11577	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11578	return rc;
11579	}
11580	}
11581	#endif
11582
11583	VMXTRANSIENT VmxTransient;
11584	RT_ZERO(VmxTransient);
11585	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11586
11587	/* Paranoia. */
11588	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11589
11590	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11591	for (;;)
11592	{
11593	Assert(!HMR0SuspendPending());
11594	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11595	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11596
11597	/*
11598	* Preparatory work for running nested-guest code, this may force us to
11599	* return to ring-3.
11600	*
11601	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11602	*/
11603	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11604	if (rcStrict != VINF_SUCCESS)
11605	break;
11606
11607	/* Interrupts are disabled at this point! */
11608	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11609	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11610	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11611	/* Interrupts are re-enabled at this point! */
11612
11613	/*
11614	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11615	*/
11616	if (RT_SUCCESS(rcRun))
11617	{ /* very likely */ }
11618	else
11619	{
11620	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11621	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11622	return rcRun;
11623	}
11624
11625	/*
11626	* Profile the VM-exit.
11627	*/
11628	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11629	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11630	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11631	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11632	HMVMX_START_EXIT_DISPATCH_PROF();
11633
11634	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11635
11636	/*
11637	* Handle the VM-exit.
11638	*/
11639	#ifdef HMVMX_USE_FUNCTION_TABLE
11640	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11641	#else
11642	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11643	#endif
11644	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11645	if (rcStrict == VINF_SUCCESS)
11646	{
11647	if (++(*pcLoops) <= cMaxResumeLoops)
11648	continue;
11649	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11650	rcStrict = VINF_EM_RAW_INTERRUPT;
11651	}
11652	break;
11653	}
11654
11655	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11656	return rcStrict;
11657	}
11658
11659
11660	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11661	/**
11662	* Runs the nested-guest code using hardware-assisted VMX.
11663	*
11664	* @returns VBox status code.
11665	* @param pVCpu The cross context virtual CPU structure.
11666	* @param pcLoops Pointer to the number of executed loops.
11667	*
11668	* @sa hmR0VmxRunGuestCodeNormal.
11669	*/
11670	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11671	{
11672	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11673	Assert(pcLoops);
11674	Assert(*pcLoops <= cMaxResumeLoops);
11675	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11676
11677	/*
11678	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11679	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11680	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11681	*/
11682	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11683	{
11684	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11685	if (RT_SUCCESS(rc))
11686	{ /* likely */ }
11687	else
11688	{
11689	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11690	return rc;
11691	}
11692	}
11693
11694	VMXTRANSIENT VmxTransient;
11695	RT_ZERO(VmxTransient);
11696	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11697	VmxTransient.fIsNestedGuest = true;
11698
11699	/* Paranoia. */
11700	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11701
11702	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11703	for (;;)
11704	{
11705	Assert(!HMR0SuspendPending());
11706	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11707	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11708
11709	/*
11710	* Preparatory work for running guest code, this may force us to
11711	* return to ring-3.
11712	*
11713	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11714	*/
11715	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11716	if (rcStrict != VINF_SUCCESS)
11717	break;
11718
11719	/* Interrupts are disabled at this point! */
11720	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11721	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11722	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11723	/* Interrupts are re-enabled at this point! */
11724
11725	/*
11726	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11727	*/
11728	if (RT_SUCCESS(rcRun))
11729	{ /* very likely */ }
11730	else
11731	{
11732	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11733	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11734	return rcRun;
11735	}
11736
11737	/*
11738	* Profile the VM-exit.
11739	*/
11740	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11741	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11742	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11743	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11744	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11745	HMVMX_START_EXIT_DISPATCH_PROF();
11746
11747	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11748
11749	/*
11750	* Handle the VM-exit.
11751	*/
11752	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11753	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11754	if (rcStrict == VINF_SUCCESS)
11755	{
11756	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11757	{
11758	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11759	rcStrict = VINF_VMX_VMEXIT;
11760	}
11761	else
11762	{
11763	if (++(*pcLoops) <= cMaxResumeLoops)
11764	continue;
11765	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11766	rcStrict = VINF_EM_RAW_INTERRUPT;
11767	}
11768	}
11769	else
11770	Assert(rcStrict != VINF_VMX_VMEXIT);
11771	break;
11772	}
11773
11774	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11775	return rcStrict;
11776	}
11777	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11778
11779
11780	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11781	* probes.
11782	*
11783	* The following few functions and associated structure contains the bloat
11784	* necessary for providing detailed debug events and dtrace probes as well as
11785	* reliable host side single stepping. This works on the principle of
11786	* "subclassing" the normal execution loop and workers. We replace the loop
11787	* method completely and override selected helpers to add necessary adjustments
11788	* to their core operation.
11789	*
11790	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11791	* any performance for debug and analysis features.
11792	*
11793	* @{
11794	*/
11795
11796	/**
11797	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11798	* the debug run loop.
11799	*/
11800	typedef struct VMXRUNDBGSTATE
11801	{
11802	/** The RIP we started executing at. This is for detecting that we stepped. */
11803	uint64_t uRipStart;
11804	/** The CS we started executing with. */
11805	uint16_t uCsStart;
11806
11807	/** Whether we've actually modified the 1st execution control field. */
11808	bool fModifiedProcCtls : 1;
11809	/** Whether we've actually modified the 2nd execution control field. */
11810	bool fModifiedProcCtls2 : 1;
11811	/** Whether we've actually modified the exception bitmap. */
11812	bool fModifiedXcptBitmap : 1;
11813
11814	/** We desire the modified the CR0 mask to be cleared. */
11815	bool fClearCr0Mask : 1;
11816	/** We desire the modified the CR4 mask to be cleared. */
11817	bool fClearCr4Mask : 1;
11818	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11819	uint32_t fCpe1Extra;
11820	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11821	uint32_t fCpe1Unwanted;
11822	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11823	uint32_t fCpe2Extra;
11824	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11825	uint32_t bmXcptExtra;
11826	/** The sequence number of the Dtrace provider settings the state was
11827	* configured against. */
11828	uint32_t uDtraceSettingsSeqNo;
11829	/** VM-exits to check (one bit per VM-exit). */
11830	uint32_t bmExitsToCheck[3];
11831
11832	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11833	uint32_t fProcCtlsInitial;
11834	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11835	uint32_t fProcCtls2Initial;
11836	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11837	uint32_t bmXcptInitial;
11838	} VMXRUNDBGSTATE;
11839	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11840	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11841
11842
11843	/**
11844	* Initializes the VMXRUNDBGSTATE structure.
11845	*
11846	* @param pVCpu The cross context virtual CPU structure of the
11847	* calling EMT.
11848	* @param pVmxTransient The VMX-transient structure.
11849	* @param pDbgState The debug state to initialize.
11850	*/
11851	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11852	{
11853	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11854	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11855
11856	pDbgState->fModifiedProcCtls = false;
11857	pDbgState->fModifiedProcCtls2 = false;
11858	pDbgState->fModifiedXcptBitmap = false;
11859	pDbgState->fClearCr0Mask = false;
11860	pDbgState->fClearCr4Mask = false;
11861	pDbgState->fCpe1Extra = 0;
11862	pDbgState->fCpe1Unwanted = 0;
11863	pDbgState->fCpe2Extra = 0;
11864	pDbgState->bmXcptExtra = 0;
11865	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11866	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11867	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11868	}
11869
11870
11871	/**
11872	* Updates the VMSC fields with changes requested by @a pDbgState.
11873	*
11874	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11875	* immediately before executing guest code, i.e. when interrupts are disabled.
11876	* We don't check status codes here as we cannot easily assert or return in the
11877	* latter case.
11878	*
11879	* @param pVCpu The cross context virtual CPU structure.
11880	* @param pVmxTransient The VMX-transient structure.
11881	* @param pDbgState The debug state.
11882	*/
11883	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11884	{
11885	/*
11886	* Ensure desired flags in VMCS control fields are set.
11887	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11888	*
11889	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11890	* there should be no stale data in pCtx at this point.
11891	*/
11892	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11893	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11894	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11895	{
11896	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11897	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11898	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11899	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11900	pDbgState->fModifiedProcCtls = true;
11901	}
11902
11903	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11904	{
11905	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11906	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11907	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11908	pDbgState->fModifiedProcCtls2 = true;
11909	}
11910
11911	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11912	{
11913	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11914	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11915	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11916	pDbgState->fModifiedXcptBitmap = true;
11917	}
11918
11919	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11920	{
11921	pVmcsInfo->u64Cr0Mask = 0;
11922	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11923	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11924	}
11925
11926	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11927	{
11928	pVmcsInfo->u64Cr4Mask = 0;
11929	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11930	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11931	}
11932
11933	NOREF(pVCpu);
11934	}
11935
11936
11937	/**
11938	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11939	* re-entry next time around.
11940	*
11941	* @returns Strict VBox status code (i.e. informational status codes too).
11942	* @param pVCpu The cross context virtual CPU structure.
11943	* @param pVmxTransient The VMX-transient structure.
11944	* @param pDbgState The debug state.
11945	* @param rcStrict The return code from executing the guest using single
11946	* stepping.
11947	*/
11948	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11949	VBOXSTRICTRC rcStrict)
11950	{
11951	/*
11952	* Restore VM-exit control settings as we may not reenter this function the
11953	* next time around.
11954	*/
11955	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11956
11957	/* We reload the initial value, trigger what we can of recalculations the
11958	next time around. From the looks of things, that's all that's required atm. */
11959	if (pDbgState->fModifiedProcCtls)
11960	{
11961	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11962	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11963	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11964	AssertRCReturn(rc2, rc2);
11965	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11966	}
11967
11968	/* We're currently the only ones messing with this one, so just restore the
11969	cached value and reload the field. */
11970	if ( pDbgState->fModifiedProcCtls2
11971	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11972	{
11973	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11974	AssertRCReturn(rc2, rc2);
11975	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11976	}
11977
11978	/* If we've modified the exception bitmap, we restore it and trigger
11979	reloading and partial recalculation the next time around. */
11980	if (pDbgState->fModifiedXcptBitmap)
11981	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11982
11983	return rcStrict;
11984	}
11985
11986
11987	/**
11988	* Configures VM-exit controls for current DBGF and DTrace settings.
11989	*
11990	* This updates @a pDbgState and the VMCS execution control fields to reflect
11991	* the necessary VM-exits demanded by DBGF and DTrace.
11992	*
11993	* @param pVCpu The cross context virtual CPU structure.
11994	* @param pVmxTransient The VMX-transient structure. May update
11995	* fUpdatedTscOffsettingAndPreemptTimer.
11996	* @param pDbgState The debug state.
11997	*/
11998	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11999	{
12000	/*
12001	* Take down the dtrace serial number so we can spot changes.
12002	*/
12003	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
12004	ASMCompilerBarrier();
12005
12006	/*
12007	* We'll rebuild most of the middle block of data members (holding the
12008	* current settings) as we go along here, so start by clearing it all.
12009	*/
12010	pDbgState->bmXcptExtra = 0;
12011	pDbgState->fCpe1Extra = 0;
12012	pDbgState->fCpe1Unwanted = 0;
12013	pDbgState->fCpe2Extra = 0;
12014	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
12015	pDbgState->bmExitsToCheck[i] = 0;
12016
12017	/*
12018	* Software interrupts (INT XXh) - no idea how to trigger these...
12019	*/
12020	PVM pVM = pVCpu->CTX_SUFF(pVM);
12021	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
12022	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
12023	{
12024	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12025	}
12026
12027	/*
12028	* INT3 breakpoints - triggered by #BP exceptions.
12029	*/
12030	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
12031	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12032
12033	/*
12034	* Exception bitmap and XCPT events+probes.
12035	*/
12036	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
12037	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
12038	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
12039
12040	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
12041	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
12042	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
12043	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
12044	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
12045	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
12046	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
12047	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
12048	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
12049	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
12050	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
12051	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
12052	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
12053	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
12054	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
12055	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
12056	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
12057	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
12058
12059	if (pDbgState->bmXcptExtra)
12060	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
12061
12062	/*
12063	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
12064	*
12065	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
12066	* So, when adding/changing/removing please don't forget to update it.
12067	*
12068	* Some of the macros are picking up local variables to save horizontal space,
12069	* (being able to see it in a table is the lesser evil here).
12070	*/
12071	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
12072	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
12073	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
12074	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
12075	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12076	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12077	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12078	} else do { } while (0)
12079	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
12080	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12081	{ \
12082	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
12083	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12084	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12085	} else do { } while (0)
12086	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
12087	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12088	{ \
12089	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
12090	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12091	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12092	} else do { } while (0)
12093	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
12094	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
12095	{ \
12096	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
12097	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
12098	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
12099	} else do { } while (0)
12100
12101	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
12102	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
12103	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
12104	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
12105	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
12106
12107	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
12108	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
12109	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
12110	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
12111	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
12112	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
12113	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
12114	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
12115	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
12116	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
12117	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
12118	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
12119	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
12120	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
12121	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
12122	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
12123	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
12124	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
12125	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
12126	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
12127	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
12128	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
12129	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
12130	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
12131	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
12132	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
12133	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
12134	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
12135	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
12136	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
12137	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
12138	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
12139	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
12140	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
12141	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
12142	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
12143
12144	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
12145	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12146	{
12147	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
12148	\| CPUMCTX_EXTRN_APIC_TPR);
12149	AssertRC(rc);
12150
12151	#if 0 /** @todo fix me */
12152	pDbgState->fClearCr0Mask = true;
12153	pDbgState->fClearCr4Mask = true;
12154	#endif
12155	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
12156	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
12157	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
12158	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
12159	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
12160	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
12161	require clearing here and in the loop if we start using it. */
12162	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
12163	}
12164	else
12165	{
12166	if (pDbgState->fClearCr0Mask)
12167	{
12168	pDbgState->fClearCr0Mask = false;
12169	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12170	}
12171	if (pDbgState->fClearCr4Mask)
12172	{
12173	pDbgState->fClearCr4Mask = false;
12174	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12175	}
12176	}
12177	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
12178	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
12179
12180	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
12181	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
12182	{
12183	/** @todo later, need to fix handler as it assumes this won't usually happen. */
12184	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
12185	}
12186	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
12187	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
12188
12189	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
12190	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
12191	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
12192	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
12193	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
12194	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
12195	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
12196	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
12197	#if 0 /** @todo too slow, fix handler. */
12198	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
12199	#endif
12200	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
12201
12202	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
12203	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
12204	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
12205	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
12206	{
12207	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12208	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
12209	}
12210	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12211	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12212	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12213	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
12214
12215	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
12216	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
12217	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
12218	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
12219	{
12220	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
12221	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
12222	}
12223	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
12224	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
12225	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
12226	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
12227
12228	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
12229	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
12230	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
12231	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
12232	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
12233	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
12234	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
12235	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
12236	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
12237	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
12238	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
12239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
12240	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
12241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
12242	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
12243	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
12244	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
12245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
12246	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
12247	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
12248	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
12249	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
12250
12251	#undef IS_EITHER_ENABLED
12252	#undef SET_ONLY_XBM_IF_EITHER_EN
12253	#undef SET_CPE1_XBM_IF_EITHER_EN
12254	#undef SET_CPEU_XBM_IF_EITHER_EN
12255	#undef SET_CPE2_XBM_IF_EITHER_EN
12256
12257	/*
12258	* Sanitize the control stuff.
12259	*/
12260	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
12261	if (pDbgState->fCpe2Extra)
12262	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
12263	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
12264	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
12265	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
12266	{
12267	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
12268	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
12269	}
12270
12271	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
12272	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
12273	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
12274	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
12275	}
12276
12277
12278	/**
12279	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
12280	* appropriate.
12281	*
12282	* The caller has checked the VM-exit against the
12283	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
12284	* already, so we don't have to do that either.
12285	*
12286	* @returns Strict VBox status code (i.e. informational status codes too).
12287	* @param pVCpu The cross context virtual CPU structure.
12288	* @param pVmxTransient The VMX-transient structure.
12289	* @param uExitReason The VM-exit reason.
12290	*
12291	* @remarks The name of this function is displayed by dtrace, so keep it short
12292	* and to the point. No longer than 33 chars long, please.
12293	*/
12294	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
12295	{
12296	/*
12297	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
12298	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
12299	*
12300	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
12301	* does. Must add/change/remove both places. Same ordering, please.
12302	*
12303	* Added/removed events must also be reflected in the next section
12304	* where we dispatch dtrace events.
12305	*/
12306	bool fDtrace1 = false;
12307	bool fDtrace2 = false;
12308	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
12309	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
12310	uint32_t uEventArg = 0;
12311	#define SET_EXIT(a_EventSubName) \
12312	do { \
12313	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12314	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12315	} while (0)
12316	#define SET_BOTH(a_EventSubName) \
12317	do { \
12318	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
12319	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
12320	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
12321	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
12322	} while (0)
12323	switch (uExitReason)
12324	{
12325	case VMX_EXIT_MTF:
12326	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12327
12328	case VMX_EXIT_XCPT_OR_NMI:
12329	{
12330	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12331	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
12332	{
12333	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12334	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
12335	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
12336	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
12337	{
12338	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
12339	{
12340	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12341	uEventArg = pVmxTransient->uExitIntErrorCode;
12342	}
12343	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
12344	switch (enmEvent1)
12345	{
12346	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
12347	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
12348	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
12349	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
12350	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
12351	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
12352	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
12353	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
12354	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
12355	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
12356	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
12357	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
12358	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
12359	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
12360	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
12361	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
12362	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
12363	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
12364	default: break;
12365	}
12366	}
12367	else
12368	AssertFailed();
12369	break;
12370
12371	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
12372	uEventArg = idxVector;
12373	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
12374	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
12375	break;
12376	}
12377	break;
12378	}
12379
12380	case VMX_EXIT_TRIPLE_FAULT:
12381	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
12382	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
12383	break;
12384	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
12385	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
12386	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
12387	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
12388	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
12389
12390	/* Instruction specific VM-exits: */
12391	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
12392	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
12393	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
12394	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
12395	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
12396	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
12397	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
12398	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
12399	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
12400	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
12401	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
12402	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
12403	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
12404	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
12405	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
12406	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
12407	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
12408	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
12409	case VMX_EXIT_MOV_CRX:
12410	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12411	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
12412	SET_BOTH(CRX_READ);
12413	else
12414	SET_BOTH(CRX_WRITE);
12415	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
12416	break;
12417	case VMX_EXIT_MOV_DRX:
12418	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12419	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
12420	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
12421	SET_BOTH(DRX_READ);
12422	else
12423	SET_BOTH(DRX_WRITE);
12424	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
12425	break;
12426	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
12427	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
12428	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
12429	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
12430	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
12431	case VMX_EXIT_GDTR_IDTR_ACCESS:
12432	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12433	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
12434	{
12435	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
12436	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
12437	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
12438	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
12439	}
12440	break;
12441
12442	case VMX_EXIT_LDTR_TR_ACCESS:
12443	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12444	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12445	{
12446	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12447	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12448	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12449	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12450	}
12451	break;
12452
12453	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12454	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12455	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12456	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12457	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12458	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12459	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12460	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12461	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12462	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12463	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12464
12465	/* Events that aren't relevant at this point. */
12466	case VMX_EXIT_EXT_INT:
12467	case VMX_EXIT_INT_WINDOW:
12468	case VMX_EXIT_NMI_WINDOW:
12469	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12470	case VMX_EXIT_PREEMPT_TIMER:
12471	case VMX_EXIT_IO_INSTR:
12472	break;
12473
12474	/* Errors and unexpected events. */
12475	case VMX_EXIT_INIT_SIGNAL:
12476	case VMX_EXIT_SIPI:
12477	case VMX_EXIT_IO_SMI:
12478	case VMX_EXIT_SMI:
12479	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12480	case VMX_EXIT_ERR_MSR_LOAD:
12481	case VMX_EXIT_ERR_MACHINE_CHECK:
12482	case VMX_EXIT_PML_FULL:
12483	case VMX_EXIT_VIRTUALIZED_EOI:
12484	break;
12485
12486	default:
12487	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12488	break;
12489	}
12490	#undef SET_BOTH
12491	#undef SET_EXIT
12492
12493	/*
12494	* Dtrace tracepoints go first. We do them here at once so we don't
12495	* have to copy the guest state saving and stuff a few dozen times.
12496	* Down side is that we've got to repeat the switch, though this time
12497	* we use enmEvent since the probes are a subset of what DBGF does.
12498	*/
12499	if (fDtrace1 \|\| fDtrace2)
12500	{
12501	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12502	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12503	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12504	switch (enmEvent1)
12505	{
12506	/** @todo consider which extra parameters would be helpful for each probe. */
12507	case DBGFEVENT_END: break;
12508	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12509	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12510	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12511	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12512	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12513	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12514	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12515	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12516	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12517	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12518	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12519	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12520	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12521	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12522	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12523	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12524	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12525	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12526	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12527	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12528	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12529	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12530	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12531	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12532	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12533	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12534	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12535	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12536	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12537	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12538	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12539	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12540	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12541	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12542	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12543	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12544	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12545	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12546	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12547	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12548	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12549	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12550	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12551	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12552	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12553	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12554	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12555	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12556	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12557	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12558	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12559	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12560	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12561	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12562	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12563	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12564	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12565	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12566	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12567	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12568	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12569	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12570	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12571	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12572	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12573	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12574	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12575	}
12576	switch (enmEvent2)
12577	{
12578	/** @todo consider which extra parameters would be helpful for each probe. */
12579	case DBGFEVENT_END: break;
12580	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12581	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12582	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12583	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12584	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12585	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12586	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12587	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12588	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12589	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12590	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12591	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12592	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12593	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12594	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12595	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12596	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12597	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12598	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12599	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12600	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12601	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12602	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12603	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12604	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12605	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12606	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12607	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12608	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12609	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12610	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12611	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12612	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12613	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12614	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12615	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12616	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12617	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12618	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12619	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12620	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12621	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12622	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12623	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12624	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12625	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12626	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12627	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12628	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12629	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12630	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12631	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12632	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12633	}
12634	}
12635
12636	/*
12637	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12638	* the DBGF call will do a full check).
12639	*
12640	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12641	* Note! If we have to events, we prioritize the first, i.e. the instruction
12642	* one, in order to avoid event nesting.
12643	*/
12644	PVM pVM = pVCpu->CTX_SUFF(pVM);
12645	if ( enmEvent1 != DBGFEVENT_END
12646	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12647	{
12648	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12649	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12650	if (rcStrict != VINF_SUCCESS)
12651	return rcStrict;
12652	}
12653	else if ( enmEvent2 != DBGFEVENT_END
12654	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12655	{
12656	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12657	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12658	if (rcStrict != VINF_SUCCESS)
12659	return rcStrict;
12660	}
12661
12662	return VINF_SUCCESS;
12663	}
12664
12665
12666	/**
12667	* Single-stepping VM-exit filtering.
12668	*
12669	* This is preprocessing the VM-exits and deciding whether we've gotten far
12670	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12671	* handling is performed.
12672	*
12673	* @returns Strict VBox status code (i.e. informational status codes too).
12674	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12675	* @param pVmxTransient The VMX-transient structure.
12676	* @param pDbgState The debug state.
12677	*/
12678	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12679	{
12680	/*
12681	* Expensive (saves context) generic dtrace VM-exit probe.
12682	*/
12683	uint32_t const uExitReason = pVmxTransient->uExitReason;
12684	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12685	{ /* more likely */ }
12686	else
12687	{
12688	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12689	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12690	AssertRC(rc);
12691	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12692	}
12693
12694	/*
12695	* Check for host NMI, just to get that out of the way.
12696	*/
12697	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12698	{ /* normally likely */ }
12699	else
12700	{
12701	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12702	AssertRCReturn(rc2, rc2);
12703	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12704	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12705	return hmR0VmxExitHostNmi(pVCpu);
12706	}
12707
12708	/*
12709	* Check for single stepping event if we're stepping.
12710	*/
12711	if (pVCpu->hm.s.fSingleInstruction)
12712	{
12713	switch (uExitReason)
12714	{
12715	case VMX_EXIT_MTF:
12716	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12717
12718	/* Various events: */
12719	case VMX_EXIT_XCPT_OR_NMI:
12720	case VMX_EXIT_EXT_INT:
12721	case VMX_EXIT_TRIPLE_FAULT:
12722	case VMX_EXIT_INT_WINDOW:
12723	case VMX_EXIT_NMI_WINDOW:
12724	case VMX_EXIT_TASK_SWITCH:
12725	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12726	case VMX_EXIT_APIC_ACCESS:
12727	case VMX_EXIT_EPT_VIOLATION:
12728	case VMX_EXIT_EPT_MISCONFIG:
12729	case VMX_EXIT_PREEMPT_TIMER:
12730
12731	/* Instruction specific VM-exits: */
12732	case VMX_EXIT_CPUID:
12733	case VMX_EXIT_GETSEC:
12734	case VMX_EXIT_HLT:
12735	case VMX_EXIT_INVD:
12736	case VMX_EXIT_INVLPG:
12737	case VMX_EXIT_RDPMC:
12738	case VMX_EXIT_RDTSC:
12739	case VMX_EXIT_RSM:
12740	case VMX_EXIT_VMCALL:
12741	case VMX_EXIT_VMCLEAR:
12742	case VMX_EXIT_VMLAUNCH:
12743	case VMX_EXIT_VMPTRLD:
12744	case VMX_EXIT_VMPTRST:
12745	case VMX_EXIT_VMREAD:
12746	case VMX_EXIT_VMRESUME:
12747	case VMX_EXIT_VMWRITE:
12748	case VMX_EXIT_VMXOFF:
12749	case VMX_EXIT_VMXON:
12750	case VMX_EXIT_MOV_CRX:
12751	case VMX_EXIT_MOV_DRX:
12752	case VMX_EXIT_IO_INSTR:
12753	case VMX_EXIT_RDMSR:
12754	case VMX_EXIT_WRMSR:
12755	case VMX_EXIT_MWAIT:
12756	case VMX_EXIT_MONITOR:
12757	case VMX_EXIT_PAUSE:
12758	case VMX_EXIT_GDTR_IDTR_ACCESS:
12759	case VMX_EXIT_LDTR_TR_ACCESS:
12760	case VMX_EXIT_INVEPT:
12761	case VMX_EXIT_RDTSCP:
12762	case VMX_EXIT_INVVPID:
12763	case VMX_EXIT_WBINVD:
12764	case VMX_EXIT_XSETBV:
12765	case VMX_EXIT_RDRAND:
12766	case VMX_EXIT_INVPCID:
12767	case VMX_EXIT_VMFUNC:
12768	case VMX_EXIT_RDSEED:
12769	case VMX_EXIT_XSAVES:
12770	case VMX_EXIT_XRSTORS:
12771	{
12772	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12773	AssertRCReturn(rc, rc);
12774	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12775	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12776	return VINF_EM_DBG_STEPPED;
12777	break;
12778	}
12779
12780	/* Errors and unexpected events: */
12781	case VMX_EXIT_INIT_SIGNAL:
12782	case VMX_EXIT_SIPI:
12783	case VMX_EXIT_IO_SMI:
12784	case VMX_EXIT_SMI:
12785	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12786	case VMX_EXIT_ERR_MSR_LOAD:
12787	case VMX_EXIT_ERR_MACHINE_CHECK:
12788	case VMX_EXIT_PML_FULL:
12789	case VMX_EXIT_VIRTUALIZED_EOI:
12790	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12791	break;
12792
12793	default:
12794	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12795	break;
12796	}
12797	}
12798
12799	/*
12800	* Check for debugger event breakpoints and dtrace probes.
12801	*/
12802	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12803	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12804	{
12805	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12806	if (rcStrict != VINF_SUCCESS)
12807	return rcStrict;
12808	}
12809
12810	/*
12811	* Normal processing.
12812	*/
12813	#ifdef HMVMX_USE_FUNCTION_TABLE
12814	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12815	#else
12816	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12817	#endif
12818	}
12819
12820
12821	/**
12822	* Single steps guest code using hardware-assisted VMX.
12823	*
12824	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12825	* but single-stepping through the hypervisor debugger.
12826	*
12827	* @returns Strict VBox status code (i.e. informational status codes too).
12828	* @param pVCpu The cross context virtual CPU structure.
12829	* @param pcLoops Pointer to the number of executed loops.
12830	*
12831	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12832	*/
12833	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12834	{
12835	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12836	Assert(pcLoops);
12837	Assert(*pcLoops <= cMaxResumeLoops);
12838
12839	VMXTRANSIENT VmxTransient;
12840	RT_ZERO(VmxTransient);
12841	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12842
12843	/* Set HMCPU indicators. */
12844	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12845	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12846	pVCpu->hm.s.fDebugWantRdTscExit = false;
12847	pVCpu->hm.s.fUsingDebugLoop = true;
12848
12849	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12850	VMXRUNDBGSTATE DbgState;
12851	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12852	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12853
12854	/*
12855	* The loop.
12856	*/
12857	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12858	for (;;)
12859	{
12860	Assert(!HMR0SuspendPending());
12861	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12862	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12863	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12864
12865	/* Set up VM-execution controls the next two can respond to. */
12866	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12867
12868	/*
12869	* Preparatory work for running guest code, this may force us to
12870	* return to ring-3.
12871	*
12872	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12873	*/
12874	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12875	if (rcStrict != VINF_SUCCESS)
12876	break;
12877
12878	/* Interrupts are disabled at this point! */
12879	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12880
12881	/* Override any obnoxious code in the above two calls. */
12882	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12883
12884	/*
12885	* Finally execute the guest.
12886	*/
12887	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12888
12889	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12890	/* Interrupts are re-enabled at this point! */
12891
12892	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12893	if (RT_SUCCESS(rcRun))
12894	{ /* very likely */ }
12895	else
12896	{
12897	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12898	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12899	return rcRun;
12900	}
12901
12902	/* Profile the VM-exit. */
12903	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12904	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12905	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12906	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12907	HMVMX_START_EXIT_DISPATCH_PROF();
12908
12909	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12910
12911	/*
12912	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12913	*/
12914	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12915	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12916	if (rcStrict != VINF_SUCCESS)
12917	break;
12918	if (++(*pcLoops) > cMaxResumeLoops)
12919	{
12920	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12921	rcStrict = VINF_EM_RAW_INTERRUPT;
12922	break;
12923	}
12924
12925	/*
12926	* Stepping: Did the RIP change, if so, consider it a single step.
12927	* Otherwise, make sure one of the TFs gets set.
12928	*/
12929	if (fStepping)
12930	{
12931	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12932	AssertRC(rc);
12933	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12934	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12935	{
12936	rcStrict = VINF_EM_DBG_STEPPED;
12937	break;
12938	}
12939	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12940	}
12941
12942	/*
12943	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12944	*/
12945	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12946	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12947	}
12948
12949	/*
12950	* Clear the X86_EFL_TF if necessary.
12951	*/
12952	if (pVCpu->hm.s.fClearTrapFlag)
12953	{
12954	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12955	AssertRC(rc);
12956	pVCpu->hm.s.fClearTrapFlag = false;
12957	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12958	}
12959	/** @todo there seems to be issues with the resume flag when the monitor trap
12960	* flag is pending without being used. Seen early in bios init when
12961	* accessing APIC page in protected mode. */
12962
12963	/*
12964	* Restore VM-exit control settings as we may not re-enter this function the
12965	* next time around.
12966	*/
12967	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12968
12969	/* Restore HMCPU indicators. */
12970	pVCpu->hm.s.fUsingDebugLoop = false;
12971	pVCpu->hm.s.fDebugWantRdTscExit = false;
12972	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12973
12974	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12975	return rcStrict;
12976	}
12977
12978
12979	/** @} */
12980
12981
12982	/**
12983	* Checks if any expensive dtrace probes are enabled and we should go to the
12984	* debug loop.
12985	*
12986	* @returns true if we should use debug loop, false if not.
12987	*/
12988	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12989	{
12990	/* It's probably faster to OR the raw 32-bit counter variables together.
12991	Since the variables are in an array and the probes are next to one
12992	another (more or less), we have good locality. So, better read
12993	eight-nine cache lines ever time and only have one conditional, than
12994	128+ conditionals, right? */
12995	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12996	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12997	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12998	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12999	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
13000	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
13001	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
13002	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
13003	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
13004	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
13005	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
13006	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
13007	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
13008	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
13009	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
13010	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
13011	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
13012	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
13013	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
13014	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
13015	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
13016	) != 0
13017	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
13018	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
13019	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
13020	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
13021	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
13022	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
13023	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
13024	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
13025	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
13026	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
13027	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
13028	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
13029	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
13030	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
13031	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
13032	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
13033	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
13034	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
13035	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
13036	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
13037	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
13038	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
13039	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
13040	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
13041	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
13042	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
13043	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
13044	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
13045	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
13046	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
13047	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
13048	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
13049	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
13050	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
13051	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
13052	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
13053	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
13054	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
13055	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
13056	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
13057	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
13058	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
13059	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
13060	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
13061	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
13062	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
13063	) != 0
13064	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
13065	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
13066	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
13067	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
13068	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
13069	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
13070	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
13071	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
13072	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
13073	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
13074	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
13075	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
13076	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
13077	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
13078	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
13079	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
13080	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
13081	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
13082	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
13083	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
13084	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
13085	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
13086	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
13087	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
13088	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
13089	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
13090	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
13091	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
13092	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
13093	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
13094	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
13095	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
13096	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
13097	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
13098	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
13099	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
13100	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
13101	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
13102	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
13103	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
13104	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
13105	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
13106	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
13107	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
13108	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
13109	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
13110	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
13111	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
13112	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
13113	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
13114	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
13115	) != 0;
13116	}
13117
13118
13119	/**
13120	* Runs the guest using hardware-assisted VMX.
13121	*
13122	* @returns Strict VBox status code (i.e. informational status codes too).
13123	* @param pVCpu The cross context virtual CPU structure.
13124	*/
13125	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
13126	{
13127	AssertPtr(pVCpu);
13128	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13129	Assert(VMMRZCallRing3IsEnabled(pVCpu));
13130	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13131	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
13132
13133	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
13134
13135	VBOXSTRICTRC rcStrict;
13136	uint32_t cLoops = 0;
13137	for (;;)
13138	{
13139	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13140	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
13141	#else
13142	bool const fInNestedGuestMode = false;
13143	#endif
13144	if (!fInNestedGuestMode)
13145	{
13146	if ( !pVCpu->hm.s.fUseDebugLoop
13147	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
13148	&& !DBGFIsStepping(pVCpu)
13149	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
13150	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
13151	else
13152	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
13153	}
13154	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13155	else
13156	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
13157
13158	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
13159	{
13160	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
13161	continue;
13162	}
13163	if (rcStrict == VINF_VMX_VMEXIT)
13164	{
13165	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
13166	continue;
13167	}
13168	#endif
13169	break;
13170	}
13171
13172	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
13173	switch (rcLoop)
13174	{
13175	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
13176	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
13177	}
13178
13179	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
13180	if (RT_FAILURE(rc2))
13181	{
13182	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
13183	rcStrict = rc2;
13184	}
13185	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
13186	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
13187	return rcStrict;
13188	}
13189
13190
13191	#ifndef HMVMX_USE_FUNCTION_TABLE
13192	/**
13193	* Handles a guest VM-exit from hardware-assisted VMX execution.
13194	*
13195	* @returns Strict VBox status code (i.e. informational status codes too).
13196	* @param pVCpu The cross context virtual CPU structure.
13197	* @param pVmxTransient The VMX-transient structure.
13198	*/
13199	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13200	{
13201	#ifdef DEBUG_ramshankar
13202	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
13203	do { \
13204	if (a_fSave != 0) \
13205	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
13206	VBOXSTRICTRC rcStrict = a_CallExpr; \
13207	if (a_fSave != 0) \
13208	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
13209	return rcStrict; \
13210	} while (0)
13211	#else
13212	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
13213	#endif
13214	uint32_t const uExitReason = pVmxTransient->uExitReason;
13215	switch (uExitReason)
13216	{
13217	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
13218	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
13219	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
13220	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
13221	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
13222	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
13223	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
13224	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
13225	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
13226	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
13227	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
13228	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
13229	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
13230	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
13231	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
13232	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
13233	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
13234	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
13235	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
13236	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
13237	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
13238	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
13239	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
13240	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
13241	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
13242	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
13243	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
13244	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
13245	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
13246	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
13247	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13248	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
13249	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
13250	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
13251	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
13252	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
13253	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
13254	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
13255	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
13256	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
13257	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
13258	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
13259	#else
13260	case VMX_EXIT_VMCLEAR:
13261	case VMX_EXIT_VMLAUNCH:
13262	case VMX_EXIT_VMPTRLD:
13263	case VMX_EXIT_VMPTRST:
13264	case VMX_EXIT_VMREAD:
13265	case VMX_EXIT_VMRESUME:
13266	case VMX_EXIT_VMWRITE:
13267	case VMX_EXIT_VMXOFF:
13268	case VMX_EXIT_VMXON:
13269	case VMX_EXIT_INVVPID:
13270	case VMX_EXIT_INVEPT:
13271	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
13272	#endif
13273
13274	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
13275	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
13276	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
13277
13278	case VMX_EXIT_INIT_SIGNAL:
13279	case VMX_EXIT_SIPI:
13280	case VMX_EXIT_IO_SMI:
13281	case VMX_EXIT_SMI:
13282	case VMX_EXIT_ERR_MSR_LOAD:
13283	case VMX_EXIT_ERR_MACHINE_CHECK:
13284	case VMX_EXIT_PML_FULL:
13285	case VMX_EXIT_VIRTUALIZED_EOI:
13286	case VMX_EXIT_GDTR_IDTR_ACCESS:
13287	case VMX_EXIT_LDTR_TR_ACCESS:
13288	case VMX_EXIT_APIC_WRITE:
13289	case VMX_EXIT_RDRAND:
13290	case VMX_EXIT_RSM:
13291	case VMX_EXIT_VMFUNC:
13292	case VMX_EXIT_ENCLS:
13293	case VMX_EXIT_RDSEED:
13294	case VMX_EXIT_XSAVES:
13295	case VMX_EXIT_XRSTORS:
13296	case VMX_EXIT_UMWAIT:
13297	case VMX_EXIT_TPAUSE:
13298	default:
13299	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13300	}
13301	#undef VMEXIT_CALL_RET
13302	}
13303	#endif /* !HMVMX_USE_FUNCTION_TABLE */
13304
13305
13306	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13307	/**
13308	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
13309	*
13310	* @returns Strict VBox status code (i.e. informational status codes too).
13311	* @param pVCpu The cross context virtual CPU structure.
13312	* @param pVmxTransient The VMX-transient structure.
13313	*/
13314	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13315	{
13316	uint32_t const uExitReason = pVmxTransient->uExitReason;
13317	switch (uExitReason)
13318	{
13319	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
13320	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
13321	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
13322	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
13323	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
13324
13325	/*
13326	* We shouldn't direct host physical interrupts to the nested-guest.
13327	*/
13328	case VMX_EXIT_EXT_INT:
13329	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
13330
13331	/*
13332	* Instructions that cause VM-exits unconditionally or the condition is
13333	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13334	* happens, it's guaranteed to be a nested-guest VM-exit).
13335	*
13336	* - Provides VM-exit instruction length ONLY.
13337	*/
13338	case VMX_EXIT_CPUID: /* Unconditional. */
13339	case VMX_EXIT_VMCALL:
13340	case VMX_EXIT_GETSEC:
13341	case VMX_EXIT_INVD:
13342	case VMX_EXIT_XSETBV:
13343	case VMX_EXIT_VMLAUNCH:
13344	case VMX_EXIT_VMRESUME:
13345	case VMX_EXIT_VMXOFF:
13346	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
13347	case VMX_EXIT_VMFUNC:
13348	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
13349
13350	/*
13351	* Instructions that cause VM-exits unconditionally or the condition is
13352	* always is taken solely from the guest hypervisor (meaning if the VM-exit
13353	* happens, it's guaranteed to be a nested-guest VM-exit).
13354	*
13355	* - Provides VM-exit instruction length.
13356	* - Provides VM-exit information.
13357	* - Optionally provides Exit qualification.
13358	*
13359	* Since Exit qualification is 0 for all VM-exits where it is not
13360	* applicable, reading and passing it to the guest should produce
13361	* defined behavior.
13362	*
13363	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
13364	*/
13365	case VMX_EXIT_INVEPT: /* Unconditional. */
13366	case VMX_EXIT_INVVPID:
13367	case VMX_EXIT_VMCLEAR:
13368	case VMX_EXIT_VMPTRLD:
13369	case VMX_EXIT_VMPTRST:
13370	case VMX_EXIT_VMXON:
13371	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
13372	case VMX_EXIT_LDTR_TR_ACCESS:
13373	case VMX_EXIT_RDRAND:
13374	case VMX_EXIT_RDSEED:
13375	case VMX_EXIT_XSAVES:
13376	case VMX_EXIT_XRSTORS:
13377	case VMX_EXIT_UMWAIT:
13378	case VMX_EXIT_TPAUSE:
13379	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
13380
13381	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
13382	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
13383	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
13384	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
13385	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
13386	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
13387	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
13388	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
13389	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
13390	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
13391	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
13392	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
13393	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
13394	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
13395	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
13396	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
13397	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
13398	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
13399	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
13400
13401	case VMX_EXIT_PREEMPT_TIMER:
13402	{
13403	/** @todo NSTVMX: Preempt timer. */
13404	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
13405	}
13406
13407	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
13408	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
13409
13410	case VMX_EXIT_VMREAD:
13411	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
13412
13413	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
13414	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
13415
13416	case VMX_EXIT_INIT_SIGNAL:
13417	case VMX_EXIT_SIPI:
13418	case VMX_EXIT_IO_SMI:
13419	case VMX_EXIT_SMI:
13420	case VMX_EXIT_ERR_MSR_LOAD:
13421	case VMX_EXIT_ERR_MACHINE_CHECK:
13422	case VMX_EXIT_PML_FULL:
13423	case VMX_EXIT_RSM:
13424	default:
13425	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
13426	}
13427	}
13428	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13429
13430
13431	/** @name VM-exit helpers.
13432	* @{
13433	*/
13434	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13435	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13436	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13437
13438	/** Macro for VM-exits called unexpectedly. */
13439	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
13440	do { \
13441	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
13442	return VERR_VMX_UNEXPECTED_EXIT; \
13443	} while (0)
13444
13445	#ifdef VBOX_STRICT
13446	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13447	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13448	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13449
13450	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13451	do { \
13452	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13453	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13454	} while (0)
13455
13456	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13457	do { \
13458	AssertPtr((a_pVCpu)); \
13459	AssertPtr((a_pVmxTransient)); \
13460	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13461	Assert((a_pVmxTransient)->pVmcsInfo); \
13462	Assert(ASMIntAreEnabled()); \
13463	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13464	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13465	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13466	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13467	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13468	HMVMX_ASSERT_PREEMPT_CPUID(); \
13469	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13470	} while (0)
13471
13472	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13473	do { \
13474	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13475	Assert((a_pVmxTransient)->fIsNestedGuest); \
13476	} while (0)
13477
13478	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13479	do { \
13480	Log4Func(("\n")); \
13481	} while (0)
13482	#else
13483	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13484	do { \
13485	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13486	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13487	} while (0)
13488
13489	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13490	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13491
13492	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13493	#endif
13494
13495	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13496	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13497	* guests that attempted to execute a VMX instruction. */
13498	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13499	do \
13500	{ \
13501	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13502	if (rcStrictTmp == VINF_SUCCESS) \
13503	{ /* likely */ } \
13504	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13505	{ \
13506	Assert((a_pVCpu)->hm.s.Event.fPending); \
13507	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13508	return VINF_SUCCESS; \
13509	} \
13510	else \
13511	{ \
13512	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13513	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13514	} \
13515	} while (0)
13516
13517	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13518	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13519	do \
13520	{ \
13521	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13522	(a_pGCPtrEffAddr)); \
13523	if (rcStrictTmp == VINF_SUCCESS) \
13524	{ /* likely */ } \
13525	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13526	{ \
13527	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13528	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13529	NOREF(uXcptTmp); \
13530	return VINF_SUCCESS; \
13531	} \
13532	else \
13533	{ \
13534	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13535	return rcStrictTmp; \
13536	} \
13537	} while (0)
13538	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13539
13540
13541	/**
13542	* Advances the guest RIP by the specified number of bytes.
13543	*
13544	* @param pVCpu The cross context virtual CPU structure.
13545	* @param cbInstr Number of bytes to advance the RIP by.
13546	*
13547	* @remarks No-long-jump zone!!!
13548	*/
13549	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13550	{
13551	/* Advance the RIP. */
13552	pVCpu->cpum.GstCtx.rip += cbInstr;
13553	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13554
13555	/* Update interrupt inhibition. */
13556	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13557	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13558	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13559	}
13560
13561
13562	/**
13563	* Advances the guest RIP after reading it from the VMCS.
13564	*
13565	* @returns VBox status code, no informational status codes.
13566	* @param pVCpu The cross context virtual CPU structure.
13567	* @param pVmxTransient The VMX-transient structure.
13568	*
13569	* @remarks No-long-jump zone!!!
13570	*/
13571	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13572	{
13573	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13574	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13575	AssertRCReturn(rc, rc);
13576
13577	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13578	return VINF_SUCCESS;
13579	}
13580
13581
13582	/**
13583	* Handle a condition that occurred while delivering an event through the guest
13584	* IDT.
13585	*
13586	* @returns Strict VBox status code (i.e. informational status codes too).
13587	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13588	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13589	* to continue execution of the guest which will delivery the \#DF.
13590	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13591	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13592	*
13593	* @param pVCpu The cross context virtual CPU structure.
13594	* @param pVmxTransient The VMX-transient structure.
13595	*
13596	* @remarks No-long-jump zone!!!
13597	*/
13598	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13599	{
13600	/* Read the IDT vectoring info. and VM-exit interruption info. */
13601	{
13602	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13603	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13604	AssertRCReturn(rc, rc);
13605	}
13606
13607	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13608	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13609	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
13610	{
13611	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13612	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13613
13614	/*
13615	* If the event was a software interrupt (generated with INT n) or a software exception
13616	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13617	* can handle the VM-exit and continue guest execution which will re-execute the
13618	* instruction rather than re-injecting the exception, as that can cause premature
13619	* trips to ring-3 before injection and involve TRPM which currently has no way of
13620	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13621	* the problem).
13622	*/
13623	IEMXCPTRAISE enmRaise;
13624	IEMXCPTRAISEINFO fRaiseInfo;
13625	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13626	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13627	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13628	{
13629	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13630	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13631	}
13632	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
13633	{
13634	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
13635	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13636	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13637
13638	/** @todo Make AssertMsgReturn as just AssertMsg later. */
13639	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
13640	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
13641
13642	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13643
13644	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13645	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13646	{
13647	pVmxTransient->fVectoringPF = true;
13648	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13649	}
13650	}
13651	else
13652	{
13653	/*
13654	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13655	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13656	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13657	*/
13658	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13659	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13660	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13661	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13662	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13663	}
13664
13665	/*
13666	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13667	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13668	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13669	* subsequent VM-entry would fail, see @bugref{7445}.
13670	*
13671	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13672	*/
13673	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13674	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
13675	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
13676	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI)
13677	&& CPUMIsGuestNmiBlocking(pVCpu))
13678	{
13679	CPUMSetGuestNmiBlocking(pVCpu, false);
13680	}
13681
13682	switch (enmRaise)
13683	{
13684	case IEMXCPTRAISE_CURRENT_XCPT:
13685	{
13686	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
13687	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
13688	Assert(rcStrict == VINF_SUCCESS);
13689	break;
13690	}
13691
13692	case IEMXCPTRAISE_PREV_EVENT:
13693	{
13694	uint32_t u32ErrCode;
13695	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
13696	{
13697	int rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13698	AssertRCReturn(rc, rc);
13699	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13700	}
13701	else
13702	u32ErrCode = 0;
13703
13704	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13705	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13706	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13707	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13708
13709	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13710	pVCpu->hm.s.Event.u32ErrCode));
13711	Assert(rcStrict == VINF_SUCCESS);
13712	break;
13713	}
13714
13715	case IEMXCPTRAISE_REEXEC_INSTR:
13716	Assert(rcStrict == VINF_SUCCESS);
13717	break;
13718
13719	case IEMXCPTRAISE_DOUBLE_FAULT:
13720	{
13721	/*
13722	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13723	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13724	*/
13725	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13726	{
13727	pVmxTransient->fVectoringDoublePF = true;
13728	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13729	pVCpu->cpum.GstCtx.cr2));
13730	rcStrict = VINF_SUCCESS;
13731	}
13732	else
13733	{
13734	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13735	hmR0VmxSetPendingXcptDF(pVCpu);
13736	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13737	uIdtVector, uExitVector));
13738	rcStrict = VINF_HM_DOUBLE_FAULT;
13739	}
13740	break;
13741	}
13742
13743	case IEMXCPTRAISE_TRIPLE_FAULT:
13744	{
13745	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
13746	rcStrict = VINF_EM_RESET;
13747	break;
13748	}
13749
13750	case IEMXCPTRAISE_CPU_HANG:
13751	{
13752	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13753	rcStrict = VERR_EM_GUEST_CPU_HANG;
13754	break;
13755	}
13756
13757	default:
13758	{
13759	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13760	rcStrict = VERR_VMX_IPE_2;
13761	break;
13762	}
13763	}
13764	}
13765	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
13766	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
13767	&& uExitVector != X86_XCPT_DF
13768	&& hmR0VmxIsPinCtlsSet(pVCpu, pVmxTransient, VMX_PIN_CTLS_VIRT_NMI))
13769	{
13770	/*
13771	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
13772	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
13773	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
13774	*/
13775	CPUMSetGuestNmiBlocking(pVCpu, true);
13776	Log4Func(("Set NMI blocking. fValid=%RTbool uExitReason=%u\n", VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo),
13777	pVmxTransient->uExitReason));
13778	}
13779
13780	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13781	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13782	return rcStrict;
13783	}
13784
13785
13786	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13787	/**
13788	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13789	* guest attempting to execute a VMX instruction.
13790	*
13791	* @returns Strict VBox status code (i.e. informational status codes too).
13792	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13793	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13794	*
13795	* @param pVCpu The cross context virtual CPU structure.
13796	* @param uExitReason The VM-exit reason.
13797	*
13798	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13799	* @remarks No-long-jump zone!!!
13800	*/
13801	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13802	{
13803	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13804	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13805
13806	/*
13807	* The physical CPU would have already checked the CPU mode/code segment.
13808	* We shall just assert here for paranoia.
13809	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13810	*/
13811	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13812	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13813	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13814
13815	if (uExitReason == VMX_EXIT_VMXON)
13816	{
13817	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13818
13819	/*
13820	* We check CR4.VMXE because it is required to be always set while in VMX operation
13821	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13822	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13823	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13824	*/
13825	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13826	{
13827	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13828	hmR0VmxSetPendingXcptUD(pVCpu);
13829	return VINF_HM_PENDING_XCPT;
13830	}
13831	}
13832	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13833	{
13834	/*
13835	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13836	* (other than VMXON), we need to raise a #UD.
13837	*/
13838	Log4Func(("Not in VMX root mode -> #UD\n"));
13839	hmR0VmxSetPendingXcptUD(pVCpu);
13840	return VINF_HM_PENDING_XCPT;
13841	}
13842
13843	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13844	return VINF_SUCCESS;
13845	}
13846
13847	/**
13848	* Decodes the memory operand of an instruction that caused a VM-exit.
13849	*
13850	* The Exit qualification field provides the displacement field for memory
13851	* operand instructions, if any.
13852	*
13853	* @returns Strict VBox status code (i.e. informational status codes too).
13854	* @retval VINF_SUCCESS if the operand was successfully decoded.
13855	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13856	* operand.
13857	* @param pVCpu The cross context virtual CPU structure.
13858	* @param uExitInstrInfo The VM-exit instruction information field.
13859	* @param enmMemAccess The memory operand's access type (read or write).
13860	* @param GCPtrDisp The instruction displacement field, if any. For
13861	* RIP-relative addressing pass RIP + displacement here.
13862	* @param pGCPtrMem Where to store the effective destination memory address.
13863	*
13864	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13865	* virtual-8086 mode hence skips those checks while verifying if the
13866	* segment is valid.
13867	*/
13868	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13869	PRTGCPTR pGCPtrMem)
13870	{
13871	Assert(pGCPtrMem);
13872	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13873	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13874	\| CPUMCTX_EXTRN_CR0);
13875
13876	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13877	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13878	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13879
13880	VMXEXITINSTRINFO ExitInstrInfo;
13881	ExitInstrInfo.u = uExitInstrInfo;
13882	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13883	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13884	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13885	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13886	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13887	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13888	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13889	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13890	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13891
13892	/*
13893	* Validate instruction information.
13894	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13895	*/
13896	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13897	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13898	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13899	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13900	AssertLogRelMsgReturn(fIsMemOperand,
13901	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13902
13903	/*
13904	* Compute the complete effective address.
13905	*
13906	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13907	* See AMD spec. 4.5.2 "Segment Registers".
13908	*/
13909	RTGCPTR GCPtrMem = GCPtrDisp;
13910	if (fBaseRegValid)
13911	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13912	if (fIdxRegValid)
13913	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13914
13915	RTGCPTR const GCPtrOff = GCPtrMem;
13916	if ( !fIsLongMode
13917	\|\| iSegReg >= X86_SREG_FS)
13918	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13919	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13920
13921	/*
13922	* Validate effective address.
13923	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13924	*/
13925	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13926	Assert(cbAccess > 0);
13927	if (fIsLongMode)
13928	{
13929	if (X86_IS_CANONICAL(GCPtrMem))
13930	{
13931	*pGCPtrMem = GCPtrMem;
13932	return VINF_SUCCESS;
13933	}
13934
13935	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13936	* "Data Limit Checks in 64-bit Mode". */
13937	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13938	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13939	return VINF_HM_PENDING_XCPT;
13940	}
13941
13942	/*
13943	* This is a watered down version of iemMemApplySegment().
13944	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13945	* and segment CPL/DPL checks are skipped.
13946	*/
13947	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13948	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13949	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13950
13951	/* Check if the segment is present and usable. */
13952	if ( pSel->Attr.n.u1Present
13953	&& !pSel->Attr.n.u1Unusable)
13954	{
13955	Assert(pSel->Attr.n.u1DescType);
13956	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13957	{
13958	/* Check permissions for the data segment. */
13959	if ( enmMemAccess == VMXMEMACCESS_WRITE
13960	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13961	{
13962	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13963	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13964	return VINF_HM_PENDING_XCPT;
13965	}
13966
13967	/* Check limits if it's a normal data segment. */
13968	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13969	{
13970	if ( GCPtrFirst32 > pSel->u32Limit
13971	\|\| GCPtrLast32 > pSel->u32Limit)
13972	{
13973	Log4Func(("Data segment limit exceeded. "
13974	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13975	GCPtrLast32, pSel->u32Limit));
13976	if (iSegReg == X86_SREG_SS)
13977	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13978	else
13979	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13980	return VINF_HM_PENDING_XCPT;
13981	}
13982	}
13983	else
13984	{
13985	/* Check limits if it's an expand-down data segment.
13986	Note! The upper boundary is defined by the B bit, not the G bit! */
13987	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13988	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13989	{
13990	Log4Func(("Expand-down data segment limit exceeded. "
13991	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13992	GCPtrLast32, pSel->u32Limit));
13993	if (iSegReg == X86_SREG_SS)
13994	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13995	else
13996	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13997	return VINF_HM_PENDING_XCPT;
13998	}
13999	}
14000	}
14001	else
14002	{
14003	/* Check permissions for the code segment. */
14004	if ( enmMemAccess == VMXMEMACCESS_WRITE
14005	\|\| ( enmMemAccess == VMXMEMACCESS_READ
14006	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
14007	{
14008	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
14009	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
14010	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14011	return VINF_HM_PENDING_XCPT;
14012	}
14013
14014	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
14015	if ( GCPtrFirst32 > pSel->u32Limit
14016	\|\| GCPtrLast32 > pSel->u32Limit)
14017	{
14018	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
14019	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
14020	if (iSegReg == X86_SREG_SS)
14021	hmR0VmxSetPendingXcptSS(pVCpu, 0);
14022	else
14023	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14024	return VINF_HM_PENDING_XCPT;
14025	}
14026	}
14027	}
14028	else
14029	{
14030	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
14031	hmR0VmxSetPendingXcptGP(pVCpu, 0);
14032	return VINF_HM_PENDING_XCPT;
14033	}
14034
14035	*pGCPtrMem = GCPtrMem;
14036	return VINF_SUCCESS;
14037	}
14038	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
14039
14040
14041	/**
14042	* VM-exit helper for LMSW.
14043	*/
14044	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
14045	{
14046	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14047	AssertRCReturn(rc, rc);
14048
14049	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
14050	AssertMsg( rcStrict == VINF_SUCCESS
14051	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14052
14053	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14054	if (rcStrict == VINF_IEM_RAISED_XCPT)
14055	{
14056	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14057	rcStrict = VINF_SUCCESS;
14058	}
14059
14060	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14061	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14062	return rcStrict;
14063	}
14064
14065
14066	/**
14067	* VM-exit helper for CLTS.
14068	*/
14069	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
14070	{
14071	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14072	AssertRCReturn(rc, rc);
14073
14074	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
14075	AssertMsg( rcStrict == VINF_SUCCESS
14076	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14077
14078	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14079	if (rcStrict == VINF_IEM_RAISED_XCPT)
14080	{
14081	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14082	rcStrict = VINF_SUCCESS;
14083	}
14084
14085	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14086	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14087	return rcStrict;
14088	}
14089
14090
14091	/**
14092	* VM-exit helper for MOV from CRx (CRx read).
14093	*/
14094	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14095	{
14096	Assert(iCrReg < 16);
14097	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
14098
14099	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14100	AssertRCReturn(rc, rc);
14101
14102	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
14103	AssertMsg( rcStrict == VINF_SUCCESS
14104	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14105
14106	if (iGReg == X86_GREG_xSP)
14107	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14108	else
14109	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14110	#ifdef VBOX_WITH_STATISTICS
14111	switch (iCrReg)
14112	{
14113	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
14114	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
14115	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
14116	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
14117	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
14118	}
14119	#endif
14120	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
14121	return rcStrict;
14122	}
14123
14124
14125	/**
14126	* VM-exit helper for MOV to CRx (CRx write).
14127	*/
14128	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14129	{
14130	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14131	AssertRCReturn(rc, rc);
14132
14133	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
14134	AssertMsg( rcStrict == VINF_SUCCESS
14135	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14136	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14137
14138	switch (iCrReg)
14139	{
14140	case 0:
14141	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14142	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
14143	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
14144	break;
14145
14146	case 2:
14147	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
14148	/* Nothing to do here, CR2 it's not part of the VMCS. */
14149	break;
14150
14151	case 3:
14152	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
14153	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
14154	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
14155	break;
14156
14157	case 4:
14158	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
14159	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
14160	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
14161	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
14162	break;
14163
14164	case 8:
14165	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
14166	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
14167	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
14168	break;
14169
14170	default:
14171	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
14172	break;
14173	}
14174
14175	if (rcStrict == VINF_IEM_RAISED_XCPT)
14176	{
14177	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14178	rcStrict = VINF_SUCCESS;
14179	}
14180	return rcStrict;
14181	}
14182
14183
14184	/**
14185	* VM-exit exception handler for \#PF (Page-fault exception).
14186	*/
14187	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14188	{
14189	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14190	PVM pVM = pVCpu->CTX_SUFF(pVM);
14191	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14192	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14193	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14194	AssertRCReturn(rc, rc);
14195
14196	if (!pVM->hm.s.fNestedPaging)
14197	{ /* likely */ }
14198	else
14199	{
14200	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14201	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
14202	#endif
14203	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14204	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14205	{
14206	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14207	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
14208	}
14209	else
14210	{
14211	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14212	hmR0VmxSetPendingXcptDF(pVCpu);
14213	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
14214	}
14215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14216	return rc;
14217	}
14218
14219	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14220	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14221	if (pVmxTransient->fVectoringPF)
14222	{
14223	Assert(pVCpu->hm.s.Event.fPending);
14224	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14225	}
14226
14227	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14228	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14229	AssertRCReturn(rc, rc);
14230
14231	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
14232	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
14233
14234	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14235	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
14236
14237	Log4Func(("#PF: rc=%Rrc\n", rc));
14238	if (rc == VINF_SUCCESS)
14239	{
14240	/*
14241	* This is typically a shadow page table sync or a MMIO instruction. But we may have
14242	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14243	*/
14244	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14245	TRPMResetTrap(pVCpu);
14246	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14247	return rc;
14248	}
14249
14250	if (rc == VINF_EM_RAW_GUEST_TRAP)
14251	{
14252	if (!pVmxTransient->fVectoringDoublePF)
14253	{
14254	/* It's a guest page fault and needs to be reflected to the guest. */
14255	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14256	TRPMResetTrap(pVCpu);
14257	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14258	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
14259	uGstErrorCode, pVmxTransient->uExitQual);
14260	}
14261	else
14262	{
14263	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14264	TRPMResetTrap(pVCpu);
14265	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14266	hmR0VmxSetPendingXcptDF(pVCpu);
14267	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
14268	}
14269
14270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14271	return VINF_SUCCESS;
14272	}
14273
14274	TRPMResetTrap(pVCpu);
14275	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14276	return rc;
14277	}
14278
14279
14280	/**
14281	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14282	*/
14283	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14284	{
14285	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14286	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14287
14288	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14289	AssertRCReturn(rc, rc);
14290
14291	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14292	{
14293	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14294	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14295
14296	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14297	* provides VM-exit instruction length. If this causes problem later,
14298	* disassemble the instruction like it's done on AMD-V. */
14299	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14300	AssertRCReturn(rc2, rc2);
14301	return rc;
14302	}
14303
14304	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14305	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14306	return rc;
14307	}
14308
14309
14310	/**
14311	* VM-exit exception handler for \#BP (Breakpoint exception).
14312	*/
14313	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14314	{
14315	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14317
14318	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14319	AssertRCReturn(rc, rc);
14320
14321	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14322	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14323	if (rc == VINF_EM_RAW_GUEST_TRAP)
14324	{
14325	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14326	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14327	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14328	AssertRCReturn(rc, rc);
14329
14330	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14331	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14332	}
14333
14334	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14335	return rc;
14336	}
14337
14338
14339	/**
14340	* VM-exit exception handler for \#AC (alignment check exception).
14341	*/
14342	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14343	{
14344	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14345	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
14346
14347	/*
14348	* Re-inject it. We'll detect any nesting before getting here.
14349	*/
14350	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14351	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14352	AssertRCReturn(rc, rc);
14353	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14354
14355	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14356	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14357	return VINF_SUCCESS;
14358	}
14359
14360
14361	/**
14362	* VM-exit exception handler for \#DB (Debug exception).
14363	*/
14364	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14365	{
14366	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14368
14369	/*
14370	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
14371	*/
14372	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14373
14374	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14375	uint64_t const uDR6 = X86_DR6_INIT_VAL
14376	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
14377	\| X86_DR6_BD \| X86_DR6_BS));
14378
14379	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14380	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14381	Log6Func(("rc=%Rrc\n", rc));
14382	if (rc == VINF_EM_RAW_GUEST_TRAP)
14383	{
14384	/*
14385	* The exception was for the guest. Update DR6, DR7.GD and
14386	* IA32_DEBUGCTL.LBR before forwarding it.
14387	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
14388	*/
14389	VMMRZCallRing3Disable(pVCpu);
14390	HM_DISABLE_PREEMPT(pVCpu);
14391
14392	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14393	pCtx->dr[6] \|= uDR6;
14394	if (CPUMIsGuestDebugStateActive(pVCpu))
14395	ASMSetDR6(pCtx->dr[6]);
14396
14397	HM_RESTORE_PREEMPT();
14398	VMMRZCallRing3Enable(pVCpu);
14399
14400	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14401	AssertRCReturn(rc, rc);
14402
14403	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14404	pCtx->dr[7] &= ~X86_DR7_GD;
14405
14406	/* Paranoia. */
14407	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14408	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14409
14410	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14411	AssertRCReturn(rc, rc);
14412
14413	/*
14414	* Raise #DB in the guest.
14415	*
14416	* It is important to reflect exactly what the VM-exit gave us (preserving the
14417	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14418	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14419	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14420	*
14421	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14422	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14423	*/
14424	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14425	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14426	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14427	AssertRCReturn(rc, rc);
14428	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14429	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14430	return VINF_SUCCESS;
14431	}
14432
14433	/*
14434	* Not a guest trap, must be a hypervisor related debug event then.
14435	* Update DR6 in case someone is interested in it.
14436	*/
14437	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14438	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14439	CPUMSetHyperDR6(pVCpu, uDR6);
14440
14441	return rc;
14442	}
14443
14444
14445	/**
14446	* Hacks its way around the lovely mesa driver's backdoor accesses.
14447	*
14448	* @sa hmR0SvmHandleMesaDrvGp.
14449	*/
14450	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14451	{
14452	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14453	RT_NOREF(pCtx);
14454
14455	/* For now we'll just skip the instruction. */
14456	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14457	}
14458
14459
14460	/**
14461	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14462	* backdoor logging w/o checking what it is running inside.
14463	*
14464	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14465	* backdoor port and magic numbers loaded in registers.
14466	*
14467	* @returns true if it is, false if it isn't.
14468	* @sa hmR0SvmIsMesaDrvGp.
14469	*/
14470	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14471	{
14472	/* 0xed: IN eAX,dx */
14473	uint8_t abInstr[1];
14474	if (pVmxTransient->cbInstr != sizeof(abInstr))
14475	return false;
14476
14477	/* Check that it is #GP(0). */
14478	if (pVmxTransient->uExitIntErrorCode != 0)
14479	return false;
14480
14481	/* Check magic and port. */
14482	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14483	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14484	if (pCtx->rax != UINT32_C(0x564d5868))
14485	return false;
14486	if (pCtx->dx != UINT32_C(0x5658))
14487	return false;
14488
14489	/* Flat ring-3 CS. */
14490	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14491	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14492	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14493	if (pCtx->cs.Attr.n.u2Dpl != 3)
14494	return false;
14495	if (pCtx->cs.u64Base != 0)
14496	return false;
14497
14498	/* Check opcode. */
14499	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14500	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14501	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14502	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14503	if (RT_FAILURE(rc))
14504	return false;
14505	if (abInstr[0] != 0xed)
14506	return false;
14507
14508	return true;
14509	}
14510
14511	/**
14512	* VM-exit exception handler for \#GP (General-protection exception).
14513	*
14514	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14515	*/
14516	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14517	{
14518	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14520
14521	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14522	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14523	if (pVmcsInfo->RealMode.fRealOnV86Active)
14524	{ /* likely */ }
14525	else
14526	{
14527	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14528	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14529	#endif
14530	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14531	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14532	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14533	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14534	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14535	AssertRCReturn(rc, rc);
14536	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14537	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14538
14539	if ( pVmxTransient->fIsNestedGuest
14540	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14541	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14542	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14543	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14544	else
14545	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14546	return rc;
14547	}
14548
14549	Assert(CPUMIsGuestInRealModeEx(pCtx));
14550	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14551	Assert(!pVmxTransient->fIsNestedGuest);
14552
14553	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14554	AssertRCReturn(rc, rc);
14555
14556	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14557	if (rcStrict == VINF_SUCCESS)
14558	{
14559	if (!CPUMIsGuestInRealModeEx(pCtx))
14560	{
14561	/*
14562	* The guest is no longer in real-mode, check if we can continue executing the
14563	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14564	*/
14565	pVmcsInfo->RealMode.fRealOnV86Active = false;
14566	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14567	{
14568	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14569	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14570	}
14571	else
14572	{
14573	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14574	rcStrict = VINF_EM_RESCHEDULE;
14575	}
14576	}
14577	else
14578	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14579	}
14580	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14581	{
14582	rcStrict = VINF_SUCCESS;
14583	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14584	}
14585	return VBOXSTRICTRC_VAL(rcStrict);
14586	}
14587
14588
14589	/**
14590	* VM-exit exception handler wrapper for generic exceptions.
14591	*
14592	* This simply re-injects the exception back into the VM without any special
14593	* processing.
14594	*/
14595	static VBOXSTRICTRC hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14596	{
14597	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14598
14599	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14600	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14601	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14602	("uVector=%#x u32XcptBitmap=%#X32\n",
14603	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14604	NOREF(pVmcsInfo);
14605	#endif
14606
14607	/*
14608	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14609	* would have been handled while checking exits due to event delivery.
14610	*/
14611	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14612	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14613	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14614	AssertRCReturn(rc, rc);
14615
14616	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14617
14618	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14619	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14620	AssertRCReturn(rc, rc);
14621	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14622	#endif
14623
14624	#ifdef VBOX_WITH_STATISTICS
14625	switch (uVector)
14626	{
14627	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14628	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14629	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14630	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14631	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14632	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14633	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14634	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14635	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14636	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14637	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14638	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14639	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14640	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14641	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14642	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14643	default:
14644	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14645	break;
14646	}
14647	#endif
14648
14649	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14650	Assert( uVector != X86_XCPT_PF
14651	\|\| VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo) != VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
14652	NOREF(uVector);
14653
14654	/* Re-inject the original exception into the guest. */
14655	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14656	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14657	return VINF_SUCCESS;
14658	}
14659	/** @} */
14660
14661
14662	/** @name VM-exit handlers.
14663	* @{
14664	*/
14665	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14666	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14667	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14668
14669	/**
14670	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14671	*/
14672	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14673	{
14674	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14676	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14677	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14678	return VINF_SUCCESS;
14679	return VINF_EM_RAW_INTERRUPT;
14680	}
14681
14682
14683	/**
14684	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14685	* VM-exit.
14686	*/
14687	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14688	{
14689	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14690	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14691
14692	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14693	AssertRCReturn(rc, rc);
14694
14695	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14696	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14697
14698	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14699	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14700	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14701	NOREF(pVmcsInfo);
14702
14703	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
14704	{
14705	/*
14706	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14707	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14708	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14709	*
14710	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14711	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
14712	*/
14713	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14714	return hmR0VmxExitHostNmi(pVCpu);
14715	}
14716
14717	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14718	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14719	if (RT_UNLIKELY(rcStrict == VINF_SUCCESS))
14720	{ /* likely */ }
14721	else
14722	{
14723	if (rcStrict == VINF_HM_DOUBLE_FAULT)
14724	{
14725	Assert(pVCpu->hm.s.Event.fPending);
14726	rcStrict = VINF_SUCCESS;
14727	}
14728	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14729	return rcStrict;
14730	}
14731
14732	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
14733	uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
14734	switch (uIntType)
14735	{
14736	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
14737	Assert(uVector == X86_XCPT_DB);
14738	RT_FALL_THRU();
14739	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
14740	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14741	RT_FALL_THRU();
14742	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14743	{
14744	/*
14745	* If there's any exception caused as a result of event injection, the resulting
14746	* secondary/final execption will be pending, we shall continue guest execution
14747	* after injecting the event. The page-fault case is complicated and we manually
14748	* handle any currently pending event in hmR0VmxExitXcptPF.
14749	*/
14750	if (!pVCpu->hm.s.Event.fPending)
14751	{ /* likely */ }
14752	else if (uVector != X86_XCPT_PF)
14753	{
14754	rcStrict = VINF_SUCCESS;
14755	break;
14756	}
14757
14758	switch (uVector)
14759	{
14760	case X86_XCPT_PF: rcStrict = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
14761	case X86_XCPT_GP: rcStrict = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
14762	case X86_XCPT_MF: rcStrict = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
14763	case X86_XCPT_DB: rcStrict = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
14764	case X86_XCPT_BP: rcStrict = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
14765	case X86_XCPT_AC: rcStrict = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
14766	default:
14767	rcStrict = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient);
14768	break;
14769	}
14770	break;
14771	}
14772
14773	default:
14774	{
14775	pVCpu->hm.s.u32HMError = uExitIntInfo;
14776	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14777	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
14778	break;
14779	}
14780	}
14781	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14782	return rcStrict;
14783	}
14784
14785
14786	/**
14787	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14788	*/
14789	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14790	{
14791	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14792
14793	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14794	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14795	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14796	AssertRCReturn(rc, rc);
14797
14798	/* Evaluate and deliver pending events and resume guest execution. */
14799	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14800	return VINF_SUCCESS;
14801	}
14802
14803
14804	/**
14805	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14806	*/
14807	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14808	{
14809	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14810
14811	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14812	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14813	{
14814	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14815	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14816	}
14817
14818	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14819
14820	/*
14821	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14822	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14823	*/
14824	uint32_t fIntrState;
14825	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14826	AssertRCReturn(rc, rc);
14827	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14828	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14829	{
14830	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14831	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14832
14833	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14834	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14835	AssertRCReturn(rc, rc);
14836	}
14837
14838	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14839	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14840	AssertRCReturn(rc, rc);
14841
14842	/* Evaluate and deliver pending events and resume guest execution. */
14843	return VINF_SUCCESS;
14844	}
14845
14846
14847	/**
14848	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14849	*/
14850	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14851	{
14852	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14853	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14854	}
14855
14856
14857	/**
14858	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14859	*/
14860	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14861	{
14862	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14863	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14864	}
14865
14866
14867	/**
14868	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14869	*/
14870	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14871	{
14872	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14873
14874	/*
14875	* Get the state we need and update the exit history entry.
14876	*/
14877	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14878	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14879	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14880	AssertRCReturn(rc, rc);
14881
14882	VBOXSTRICTRC rcStrict;
14883	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14884	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14885	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14886	if (!pExitRec)
14887	{
14888	/*
14889	* Regular CPUID instruction execution.
14890	*/
14891	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14892	if (rcStrict == VINF_SUCCESS)
14893	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14894	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14895	{
14896	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14897	rcStrict = VINF_SUCCESS;
14898	}
14899	}
14900	else
14901	{
14902	/*
14903	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14904	*/
14905	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14906	AssertRCReturn(rc2, rc2);
14907
14908	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14909	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14910
14911	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14912	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14913
14914	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14915	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14916	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14917	}
14918	return rcStrict;
14919	}
14920
14921
14922	/**
14923	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14924	*/
14925	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14926	{
14927	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14928
14929	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14930	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14931	AssertRCReturn(rc, rc);
14932
14933	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14934	return VINF_EM_RAW_EMULATE_INSTR;
14935
14936	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14937	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14938	}
14939
14940
14941	/**
14942	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14943	*/
14944	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14945	{
14946	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14947
14948	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14949	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14950	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14951	AssertRCReturn(rc, rc);
14952
14953	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
14954	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14955	{
14956	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14957	we must reset offsetting on VM-entry. See @bugref{6634}. */
14958	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14959	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14960	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14961	}
14962	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14963	{
14964	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14965	rcStrict = VINF_SUCCESS;
14966	}
14967	return rcStrict;
14968	}
14969
14970
14971	/**
14972	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14973	*/
14974	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14975	{
14976	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14977
14978	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14979	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14980	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14981	AssertRCReturn(rc, rc);
14982
14983	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
14984	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14985	{
14986	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14987	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14988	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14989	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14990	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14991	}
14992	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14993	{
14994	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14995	rcStrict = VINF_SUCCESS;
14996	}
14997	return rcStrict;
14998	}
14999
15000
15001	/**
15002	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
15003	*/
15004	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15005	{
15006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15007
15008	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15009	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
15010	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
15011	AssertRCReturn(rc, rc);
15012
15013	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15014	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
15015	if (RT_LIKELY(rc == VINF_SUCCESS))
15016	{
15017	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15018	Assert(pVmxTransient->cbInstr == 2);
15019	}
15020	else
15021	{
15022	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
15023	rc = VERR_EM_INTERPRETER;
15024	}
15025	return rc;
15026	}
15027
15028
15029	/**
15030	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
15031	*/
15032	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15033	{
15034	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15035
15036	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
15037	if (EMAreHypercallInstructionsEnabled(pVCpu))
15038	{
15039	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15040	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
15041	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
15042	AssertRCReturn(rc, rc);
15043
15044	/* Perform the hypercall. */
15045	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
15046	if (rcStrict == VINF_SUCCESS)
15047	{
15048	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15049	AssertRCReturn(rc, rc);
15050	}
15051	else
15052	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
15053	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
15054	\|\| RT_FAILURE(rcStrict));
15055
15056	/* If the hypercall changes anything other than guest's general-purpose registers,
15057	we would need to reload the guest changed bits here before VM-entry. */
15058	}
15059	else
15060	Log4Func(("Hypercalls not enabled\n"));
15061
15062	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
15063	if (RT_FAILURE(rcStrict))
15064	{
15065	hmR0VmxSetPendingXcptUD(pVCpu);
15066	rcStrict = VINF_SUCCESS;
15067	}
15068
15069	return rcStrict;
15070	}
15071
15072
15073	/**
15074	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
15075	*/
15076	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15077	{
15078	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15079	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
15080
15081	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15082	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15083	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15084	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15085	AssertRCReturn(rc, rc);
15086
15087	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
15088
15089	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
15090	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15091	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15092	{
15093	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15094	rcStrict = VINF_SUCCESS;
15095	}
15096	else
15097	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
15098	VBOXSTRICTRC_VAL(rcStrict)));
15099	return rcStrict;
15100	}
15101
15102
15103	/**
15104	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
15105	*/
15106	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15107	{
15108	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15109
15110	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15111	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
15112	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15113	AssertRCReturn(rc, rc);
15114
15115	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
15116	if (rcStrict == VINF_SUCCESS)
15117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15118	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15119	{
15120	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15121	rcStrict = VINF_SUCCESS;
15122	}
15123
15124	return rcStrict;
15125	}
15126
15127
15128	/**
15129	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
15130	*/
15131	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15132	{
15133	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15134
15135	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15136	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15137	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15138	AssertRCReturn(rc, rc);
15139
15140	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
15141	if (RT_SUCCESS(rcStrict))
15142	{
15143	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15144	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
15145	rcStrict = VINF_SUCCESS;
15146	}
15147
15148	return rcStrict;
15149	}
15150
15151
15152	/**
15153	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
15154	* VM-exit.
15155	*/
15156	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15157	{
15158	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15159	return VINF_EM_RESET;
15160	}
15161
15162
15163	/**
15164	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
15165	*/
15166	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15167	{
15168	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15169
15170	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15171	AssertRCReturn(rc, rc);
15172
15173	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
15174	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
15175	rc = VINF_SUCCESS;
15176	else
15177	rc = VINF_EM_HALT;
15178
15179	if (rc != VINF_SUCCESS)
15180	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
15181	return rc;
15182	}
15183
15184
15185	/**
15186	* VM-exit handler for instructions that result in a \#UD exception delivered to
15187	* the guest.
15188	*/
15189	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15190	{
15191	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15192	hmR0VmxSetPendingXcptUD(pVCpu);
15193	return VINF_SUCCESS;
15194	}
15195
15196
15197	/**
15198	* VM-exit handler for expiry of the VMX-preemption timer.
15199	*/
15200	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15201	{
15202	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15203
15204	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
15205	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15206
15207	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
15208	PVM pVM = pVCpu->CTX_SUFF(pVM);
15209	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
15210	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
15211	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
15212	}
15213
15214
15215	/**
15216	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
15217	*/
15218	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15219	{
15220	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15221
15222	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15223	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15224	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
15225	AssertRCReturn(rc, rc);
15226
15227	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
15228	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15229	: HM_CHANGED_RAISED_XCPT_MASK);
15230
15231	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15232	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
15233
15234	return rcStrict;
15235	}
15236
15237
15238	/**
15239	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
15240	*/
15241	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15242	{
15243	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15244	/** @todo Use VM-exit instruction information. */
15245	return VERR_EM_INTERPRETER;
15246	}
15247
15248
15249	/**
15250	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
15251	* VM-exit.
15252	*/
15253	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15254	{
15255	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15256	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15257	AssertRCReturn(rc, rc);
15258
15259	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
15260	if (RT_FAILURE(rc))
15261	return rc;
15262
15263	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
15264	NOREF(uInvalidReason);
15265
15266	#ifdef VBOX_STRICT
15267	uint32_t fIntrState;
15268	RTHCUINTREG uHCReg;
15269	uint64_t u64Val;
15270	uint32_t u32Val;
15271	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
15272	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
15273	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
15274	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
15275	AssertRCReturn(rc, rc);
15276
15277	Log4(("uInvalidReason %u\n", uInvalidReason));
15278	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
15279	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
15280	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
15281	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
15282
15283	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
15284	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
15285	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
15286	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
15287	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
15288	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15289	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
15290	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
15291	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
15292	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
15293	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
15294	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
15295
15296	hmR0DumpRegs(pVCpu);
15297	#endif
15298
15299	return VERR_VMX_INVALID_GUEST_STATE;
15300	}
15301
15302	/**
15303	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
15304	*/
15305	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15306	{
15307	/*
15308	* Cummulative notes of all recognized but unexpected VM-exits.
15309	*
15310	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
15311	* nested-paging is used.
15312	*
15313	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
15314	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
15315	* this function (and thereby stop VM execution) for handling such instructions.
15316	*
15317	*
15318	* VMX_EXIT_INIT_SIGNAL:
15319	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
15320	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
15321	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
15322	*
15323	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
15324	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
15325	* See Intel spec. "23.8 Restrictions on VMX operation".
15326	*
15327	* VMX_EXIT_SIPI:
15328	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
15329	* activity state is used. We don't make use of it as our guests don't have direct
15330	* access to the host local APIC.
15331	*
15332	* See Intel spec. 25.3 "Other Causes of VM-exits".
15333	*
15334	* VMX_EXIT_IO_SMI:
15335	* VMX_EXIT_SMI:
15336	* This can only happen if we support dual-monitor treatment of SMI, which can be
15337	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
15338	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
15339	* VMX root mode or receive an SMI. If we get here, something funny is going on.
15340	*
15341	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
15342	* See Intel spec. 25.3 "Other Causes of VM-Exits"
15343	*
15344	* VMX_EXIT_ERR_MSR_LOAD:
15345	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
15346	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
15347	* execution.
15348	*
15349	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
15350	*
15351	* VMX_EXIT_ERR_MACHINE_CHECK:
15352	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
15353	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
15354	* #MC exception abort class exception is raised. We thus cannot assume a
15355	* reasonable chance of continuing any sort of execution and we bail.
15356	*
15357	* See Intel spec. 15.1 "Machine-check Architecture".
15358	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
15359	*
15360	* VMX_EXIT_PML_FULL:
15361	* VMX_EXIT_VIRTUALIZED_EOI:
15362	* VMX_EXIT_APIC_WRITE:
15363	* We do not currently support any of these features and thus they are all unexpected
15364	* VM-exits.
15365	*
15366	* VMX_EXIT_GDTR_IDTR_ACCESS:
15367	* VMX_EXIT_LDTR_TR_ACCESS:
15368	* VMX_EXIT_RDRAND:
15369	* VMX_EXIT_RSM:
15370	* VMX_EXIT_VMFUNC:
15371	* VMX_EXIT_ENCLS:
15372	* VMX_EXIT_RDSEED:
15373	* VMX_EXIT_XSAVES:
15374	* VMX_EXIT_XRSTORS:
15375	* VMX_EXIT_UMWAIT:
15376	* VMX_EXIT_TPAUSE:
15377	* These VM-exits are -not- caused unconditionally by execution of the corresponding
15378	* instruction. Any VM-exit for these instructions indicate a hardware problem,
15379	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
15380	*
15381	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
15382	*/
15383	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15384	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
15385	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15386	}
15387
15388
15389	/**
15390	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15391	*/
15392	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15393	{
15394	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15395
15396	/** @todo Optimize this: We currently drag in in the whole MSR state
15397	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15398	* MSRs required. That would require changes to IEM and possibly CPUM too.
15399	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15400	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15401	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15402	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15403	switch (idMsr)
15404	{
15405	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15406	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15407	}
15408
15409	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15410	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15411	AssertRCReturn(rc, rc);
15412
15413	Log4Func(("ecx=%#RX32\n", idMsr));
15414
15415	#ifdef VBOX_STRICT
15416	Assert(!pVmxTransient->fIsNestedGuest);
15417	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15418	{
15419	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15420	&& idMsr != MSR_K6_EFER)
15421	{
15422	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15423	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15424	}
15425	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15426	{
15427	Assert(pVmcsInfo->pvMsrBitmap);
15428	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15429	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15430	{
15431	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15432	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15433	}
15434	}
15435	}
15436	#endif
15437
15438	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15440	if (rcStrict == VINF_SUCCESS)
15441	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15442	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15443	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15444	{
15445	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15446	rcStrict = VINF_SUCCESS;
15447	}
15448	else
15449	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15450
15451	return rcStrict;
15452	}
15453
15454
15455	/**
15456	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15457	*/
15458	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15459	{
15460	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15461
15462	/** @todo Optimize this: We currently drag in in the whole MSR state
15463	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15464	* MSRs required. That would require changes to IEM and possibly CPUM too.
15465	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15466	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15467	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15468
15469	/*
15470	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15471	* Although we don't need to fetch the base as it will be overwritten shortly, while
15472	* loading guest-state we would also load the entire segment register including limit
15473	* and attributes and thus we need to load them here.
15474	*/
15475	switch (idMsr)
15476	{
15477	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15478	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15479	}
15480
15481	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15482	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15483	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15484	AssertRCReturn(rc, rc);
15485
15486	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15487
15488	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15490
15491	if (rcStrict == VINF_SUCCESS)
15492	{
15493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15494
15495	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15496	if ( idMsr == MSR_IA32_APICBASE
15497	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15498	&& idMsr <= MSR_IA32_X2APIC_END))
15499	{
15500	/*
15501	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15502	* When full APIC register virtualization is implemented we'll have to make
15503	* sure APIC state is saved from the VMCS before IEM changes it.
15504	*/
15505	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15506	}
15507	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15508	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15509	else if (idMsr == MSR_K6_EFER)
15510	{
15511	/*
15512	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15513	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15514	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15515	*/
15516	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15517	}
15518
15519	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15520	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15521	{
15522	switch (idMsr)
15523	{
15524	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15525	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15526	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15527	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15528	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15529	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15530	default:
15531	{
15532	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15533	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15534	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15535	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15536	break;
15537	}
15538	}
15539	}
15540	#ifdef VBOX_STRICT
15541	else
15542	{
15543	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15544	switch (idMsr)
15545	{
15546	case MSR_IA32_SYSENTER_CS:
15547	case MSR_IA32_SYSENTER_EIP:
15548	case MSR_IA32_SYSENTER_ESP:
15549	case MSR_K8_FS_BASE:
15550	case MSR_K8_GS_BASE:
15551	{
15552	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15553	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15554	}
15555
15556	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15557	default:
15558	{
15559	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15560	{
15561	/* EFER MSR writes are always intercepted. */
15562	if (idMsr != MSR_K6_EFER)
15563	{
15564	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15565	idMsr));
15566	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15567	}
15568	}
15569
15570	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15571	{
15572	Assert(pVmcsInfo->pvMsrBitmap);
15573	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15574	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15575	{
15576	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15577	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15578	}
15579	}
15580	break;
15581	}
15582	}
15583	}
15584	#endif /* VBOX_STRICT */
15585	}
15586	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15587	{
15588	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15589	rcStrict = VINF_SUCCESS;
15590	}
15591	else
15592	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15593
15594	return rcStrict;
15595	}
15596
15597
15598	/**
15599	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15600	*/
15601	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15602	{
15603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15604
15605	/** @todo The guest has likely hit a contended spinlock. We might want to
15606	* poke a schedule different guest VCPU. */
15607	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15608	if (RT_SUCCESS(rc))
15609	return VINF_EM_RAW_INTERRUPT;
15610
15611	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15612	return rc;
15613	}
15614
15615
15616	/**
15617	* VM-exit handler for when the TPR value is lowered below the specified
15618	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15619	*/
15620	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15621	{
15622	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15623	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15624
15625	/*
15626	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15627	* We'll re-evaluate pending interrupts and inject them before the next VM
15628	* entry so we can just continue execution here.
15629	*/
15630	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15631	return VINF_SUCCESS;
15632	}
15633
15634
15635	/**
15636	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15637	* VM-exit.
15638	*
15639	* @retval VINF_SUCCESS when guest execution can continue.
15640	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15641	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15642	* incompatible guest state for VMX execution (real-on-v86 case).
15643	*/
15644	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15645	{
15646	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15647	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15648
15649	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15650	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15651	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15652	AssertRCReturn(rc, rc);
15653
15654	VBOXSTRICTRC rcStrict;
15655	PVM pVM = pVCpu->CTX_SUFF(pVM);
15656	uint64_t const uExitQual = pVmxTransient->uExitQual;
15657	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15658	switch (uAccessType)
15659	{
15660	/*
15661	* MOV to CRx.
15662	*/
15663	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15664	{
15665	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15666	AssertRCReturn(rc, rc);
15667
15668	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15669	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15670	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15671	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15672
15673	/*
15674	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15675	* - When nested paging isn't used.
15676	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15677	* - We are executing in the VM debug loop.
15678	*/
15679	Assert( iCrReg != 3
15680	\|\| !pVM->hm.s.fNestedPaging
15681	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15682	\|\| pVCpu->hm.s.fUsingDebugLoop);
15683
15684	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15685	Assert( iCrReg != 8
15686	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15687
15688	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15689	AssertMsg( rcStrict == VINF_SUCCESS
15690	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15691
15692	/*
15693	* This is a kludge for handling switches back to real mode when we try to use
15694	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15695	* deal with special selector values, so we have to return to ring-3 and run
15696	* there till the selector values are V86 mode compatible.
15697	*
15698	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15699	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15700	* this function.
15701	*/
15702	if ( iCrReg == 0
15703	&& rcStrict == VINF_SUCCESS
15704	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15705	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15706	&& (uOldCr0 & X86_CR0_PE)
15707	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15708	{
15709	/** @todo Check selectors rather than returning all the time. */
15710	Assert(!pVmxTransient->fIsNestedGuest);
15711	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15712	rcStrict = VINF_EM_RESCHEDULE_REM;
15713	}
15714	break;
15715	}
15716
15717	/*
15718	* MOV from CRx.
15719	*/
15720	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15721	{
15722	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15723	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15724
15725	/*
15726	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15727	* - When nested paging isn't used.
15728	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15729	* - We are executing in the VM debug loop.
15730	*/
15731	Assert( iCrReg != 3
15732	\|\| !pVM->hm.s.fNestedPaging
15733	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15734	\|\| pVCpu->hm.s.fUsingDebugLoop);
15735
15736	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15737	Assert( iCrReg != 8
15738	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15739
15740	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15741	break;
15742	}
15743
15744	/*
15745	* CLTS (Clear Task-Switch Flag in CR0).
15746	*/
15747	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15748	{
15749	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15750	break;
15751	}
15752
15753	/*
15754	* LMSW (Load Machine-Status Word into CR0).
15755	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15756	*/
15757	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15758	{
15759	RTGCPTR GCPtrEffDst;
15760	uint8_t const cbInstr = pVmxTransient->cbInstr;
15761	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15762	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15763	if (fMemOperand)
15764	{
15765	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15766	AssertRCReturn(rc, rc);
15767	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15768	}
15769	else
15770	GCPtrEffDst = NIL_RTGCPTR;
15771	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15772	break;
15773	}
15774
15775	default:
15776	{
15777	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15778	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15779	}
15780	}
15781
15782	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15783	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15784	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15785
15786	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15787	NOREF(pVM);
15788	return rcStrict;
15789	}
15790
15791
15792	/**
15793	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15794	* VM-exit.
15795	*/
15796	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15797	{
15798	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15799	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15800
15801	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15802	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15803	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15804	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15805	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15806	\| CPUMCTX_EXTRN_EFER);
15807	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15808	AssertRCReturn(rc, rc);
15809
15810	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15811	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15812	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15813	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15814	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15815	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15816	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15817	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15818
15819	/*
15820	* Update exit history to see if this exit can be optimized.
15821	*/
15822	VBOXSTRICTRC rcStrict;
15823	PCEMEXITREC pExitRec = NULL;
15824	if ( !fGstStepping
15825	&& !fDbgStepping)
15826	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15827	!fIOString
15828	? !fIOWrite
15829	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15830	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15831	: !fIOWrite
15832	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15833	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15834	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15835	if (!pExitRec)
15836	{
15837	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15838	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15839
15840	uint32_t const cbValue = s_aIOSizes[uIOSize];
15841	uint32_t const cbInstr = pVmxTransient->cbInstr;
15842	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15843	PVM pVM = pVCpu->CTX_SUFF(pVM);
15844	if (fIOString)
15845	{
15846	/*
15847	* INS/OUTS - I/O String instruction.
15848	*
15849	* Use instruction-information if available, otherwise fall back on
15850	* interpreting the instruction.
15851	*/
15852	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15853	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15854	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15855	if (fInsOutsInfo)
15856	{
15857	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15858	AssertRCReturn(rc2, rc2);
15859	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15860	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15861	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15862	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15863	if (fIOWrite)
15864	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15865	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15866	else
15867	{
15868	/*
15869	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15870	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15871	* See Intel Instruction spec. for "INS".
15872	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15873	*/
15874	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15875	}
15876	}
15877	else
15878	rcStrict = IEMExecOne(pVCpu);
15879
15880	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15881	fUpdateRipAlready = true;
15882	}
15883	else
15884	{
15885	/*
15886	* IN/OUT - I/O instruction.
15887	*/
15888	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15889	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15890	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15891	if (fIOWrite)
15892	{
15893	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15895	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15896	&& !pCtx->eflags.Bits.u1TF)
15897	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15898	}
15899	else
15900	{
15901	uint32_t u32Result = 0;
15902	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15903	if (IOM_SUCCESS(rcStrict))
15904	{
15905	/* Save result of I/O IN instr. in AL/AX/EAX. */
15906	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15907	}
15908	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15909	&& !pCtx->eflags.Bits.u1TF)
15910	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15912	}
15913	}
15914
15915	if (IOM_SUCCESS(rcStrict))
15916	{
15917	if (!fUpdateRipAlready)
15918	{
15919	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15920	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15921	}
15922
15923	/*
15924	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15925	* while booting Fedora 17 64-bit guest.
15926	*
15927	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15928	*/
15929	if (fIOString)
15930	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15931
15932	/*
15933	* If any I/O breakpoints are armed, we need to check if one triggered
15934	* and take appropriate action.
15935	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15936	*/
15937	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15938	AssertRCReturn(rc, rc);
15939
15940	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15941	* execution engines about whether hyper BPs and such are pending. */
15942	uint32_t const uDr7 = pCtx->dr[7];
15943	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15944	&& X86_DR7_ANY_RW_IO(uDr7)
15945	&& (pCtx->cr4 & X86_CR4_DE))
15946	\|\| DBGFBpIsHwIoArmed(pVM)))
15947	{
15948	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15949
15950	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15951	VMMRZCallRing3Disable(pVCpu);
15952	HM_DISABLE_PREEMPT(pVCpu);
15953
15954	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15955
15956	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15957	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15958	{
15959	/* Raise #DB. */
15960	if (fIsGuestDbgActive)
15961	ASMSetDR6(pCtx->dr[6]);
15962	if (pCtx->dr[7] != uDr7)
15963	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15964
15965	hmR0VmxSetPendingXcptDB(pVCpu);
15966	}
15967	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15968	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15969	else if ( rcStrict2 != VINF_SUCCESS
15970	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15971	rcStrict = rcStrict2;
15972	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15973
15974	HM_RESTORE_PREEMPT();
15975	VMMRZCallRing3Enable(pVCpu);
15976	}
15977	}
15978
15979	#ifdef VBOX_STRICT
15980	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15981	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15982	Assert(!fIOWrite);
15983	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15984	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15985	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15986	Assert(fIOWrite);
15987	else
15988	{
15989	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15990	* statuses, that the VMM device and some others may return. See
15991	* IOM_SUCCESS() for guidance. */
15992	AssertMsg( RT_FAILURE(rcStrict)
15993	\|\| rcStrict == VINF_SUCCESS
15994	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15995	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15996	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15997	\|\| rcStrict == VINF_EM_RAW_TO_R3
15998	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15999	# endif
16000	}
16001	#endif
16002	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
16003	}
16004	else
16005	{
16006	/*
16007	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
16008	*/
16009	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16010	AssertRCReturn(rc2, rc2);
16011	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
16012	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
16013	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
16014	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16015	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
16016	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
16017
16018	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16019	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16020
16021	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16022	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16023	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16024	}
16025	return rcStrict;
16026	}
16027
16028
16029	/**
16030	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
16031	* VM-exit.
16032	*/
16033	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16034	{
16035	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16036
16037	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
16038	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16039	AssertRCReturn(rc, rc);
16040	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
16041	{
16042	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16043	AssertRCReturn(rc, rc);
16044	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
16045	{
16046	uint32_t uErrCode;
16047	RTGCUINTPTR GCPtrFaultAddress;
16048	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
16049	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
16050	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
16051	if (fErrorCodeValid)
16052	{
16053	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16054	AssertRCReturn(rc, rc);
16055	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
16056	}
16057	else
16058	uErrCode = 0;
16059
16060	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
16061	&& uVector == X86_XCPT_PF)
16062	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
16063	else
16064	GCPtrFaultAddress = 0;
16065
16066	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16067	AssertRCReturn(rc, rc);
16068
16069	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
16070	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
16071
16072	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
16073	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16074	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16075	}
16076	}
16077
16078	/* Fall back to the interpreter to emulate the task-switch. */
16079	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
16080	return VERR_EM_INTERPRETER;
16081	}
16082
16083
16084	/**
16085	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
16086	*/
16087	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16088	{
16089	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16090
16091	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16092	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
16093	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16094	AssertRCReturn(rc, rc);
16095	return VINF_EM_DBG_STEPPED;
16096	}
16097
16098
16099	/**
16100	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
16101	*/
16102	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16103	{
16104	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16105	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
16106
16107	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16108	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16109	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
16110	{
16111	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
16112	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16113	{
16114	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16115	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16116	}
16117	}
16118	else
16119	{
16120	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
16121	{
16122	Assert(pVCpu->hm.s.Event.fPending);
16123	rcStrict1 = VINF_SUCCESS;
16124	}
16125	return rcStrict1;
16126	}
16127
16128	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
16129	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16130	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16131	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16132	AssertRCReturn(rc, rc);
16133
16134	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
16135	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
16136	VBOXSTRICTRC rcStrict2;
16137	switch (uAccessType)
16138	{
16139	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
16140	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
16141	{
16142	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
16143	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
16144	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
16145
16146	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
16147	GCPhys &= PAGE_BASE_GC_MASK;
16148	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
16149	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
16150	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
16151
16152	PVM pVM = pVCpu->CTX_SUFF(pVM);
16153	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16154	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
16155	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
16156	CPUMCTX2CORE(pCtx), GCPhys);
16157	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
16158	if ( rcStrict2 == VINF_SUCCESS
16159	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
16160	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
16161	{
16162	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16163	\| HM_CHANGED_GUEST_APIC_TPR);
16164	rcStrict2 = VINF_SUCCESS;
16165	}
16166	break;
16167	}
16168
16169	default:
16170	{
16171	Log4Func(("uAccessType=%#x\n", uAccessType));
16172	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
16173	break;
16174	}
16175	}
16176
16177	if (rcStrict2 != VINF_SUCCESS)
16178	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
16179	return rcStrict2;
16180	}
16181
16182
16183	/**
16184	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
16185	* VM-exit.
16186	*/
16187	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16188	{
16189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16190
16191	/* We should -not- get this VM-exit if the guest's debug registers were active. */
16192	if (pVmxTransient->fWasGuestDebugStateActive)
16193	{
16194	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
16195	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
16196	}
16197
16198	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16199	if ( !pVCpu->hm.s.fSingleInstruction
16200	&& !pVmxTransient->fWasHyperDebugStateActive)
16201	{
16202	Assert(!DBGFIsStepping(pVCpu));
16203	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
16204
16205	/* Don't intercept MOV DRx any more. */
16206	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
16207	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
16208	AssertRCReturn(rc, rc);
16209
16210	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
16211	VMMRZCallRing3Disable(pVCpu);
16212	HM_DISABLE_PREEMPT(pVCpu);
16213
16214	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
16215	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
16216	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
16217
16218	HM_RESTORE_PREEMPT();
16219	VMMRZCallRing3Enable(pVCpu);
16220
16221	#ifdef VBOX_WITH_STATISTICS
16222	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16223	AssertRCReturn(rc, rc);
16224	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16226	else
16227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16228	#endif
16229	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
16230	return VINF_SUCCESS;
16231	}
16232
16233	/*
16234	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
16235	* The EFER MSR is always up-to-date.
16236	* Update the segment registers and DR7 from the CPU.
16237	*/
16238	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16239	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16240	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
16241	AssertRCReturn(rc, rc);
16242	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
16243
16244	PVM pVM = pVCpu->CTX_SUFF(pVM);
16245	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
16246	{
16247	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16248	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
16249	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
16250	if (RT_SUCCESS(rc))
16251	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
16252	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
16253	}
16254	else
16255	{
16256	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
16257	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
16258	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
16259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
16260	}
16261
16262	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
16263	if (RT_SUCCESS(rc))
16264	{
16265	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
16266	AssertRCReturn(rc2, rc2);
16267	return VINF_SUCCESS;
16268	}
16269	return rc;
16270	}
16271
16272
16273	/**
16274	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
16275	* Conditional VM-exit.
16276	*/
16277	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16278	{
16279	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16280	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16281
16282	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16283	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16284	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16285	{
16286	/*
16287	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
16288	* instruction emulation to inject the original event. Otherwise, injecting the original event
16289	* using hardware-assisted VMX would would trigger the same EPT misconfig VM-exit again.
16290	*/
16291	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
16292	{
16293	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
16294	return VINF_EM_RAW_INJECT_TRPM_EVENT;
16295	}
16296	}
16297	else
16298	{
16299	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16300	rcStrict = VINF_SUCCESS;
16301	return rcStrict;
16302	}
16303
16304	/*
16305	* Get sufficent state and update the exit history entry.
16306	*/
16307	RTGCPHYS GCPhys;
16308	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16309	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16310	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16311	AssertRCReturn(rc, rc);
16312
16313	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
16314	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
16315	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
16316	if (!pExitRec)
16317	{
16318	/*
16319	* If we succeed, resume guest execution.
16320	* If we fail in interpreting the instruction because we couldn't get the guest physical address
16321	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
16322	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
16323	* weird case. See @bugref{6043}.
16324	*/
16325	PVM pVM = pVCpu->CTX_SUFF(pVM);
16326	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16327	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
16328	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
16329	if ( rcStrict == VINF_SUCCESS
16330	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16331	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16332	{
16333	/* Successfully handled MMIO operation. */
16334	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
16335	\| HM_CHANGED_GUEST_APIC_TPR);
16336	rcStrict = VINF_SUCCESS;
16337	}
16338	}
16339	else
16340	{
16341	/*
16342	* Frequent exit or something needing probing. Call EMHistoryExec.
16343	*/
16344	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
16345	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
16346
16347	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
16348	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16349
16350	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
16351	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
16352	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
16353	}
16354	return rcStrict;
16355	}
16356
16357
16358	/**
16359	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
16360	* VM-exit.
16361	*/
16362	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16363	{
16364	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16365	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16366
16367	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
16368	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16369	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16370	{
16371	/*
16372	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16373	* we shall resolve the nested #PF and re-inject the original event.
16374	*/
16375	if (pVCpu->hm.s.Event.fPending)
16376	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16377	}
16378	else
16379	{
16380	if (rcStrict == VINF_HM_DOUBLE_FAULT)
16381	{
16382	Assert(pVCpu->hm.s.Event.fPending);
16383	rcStrict = VINF_SUCCESS;
16384	}
16385	return rcStrict;
16386	}
16387
16388	RTGCPHYS GCPhys;
16389	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16390	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
16391	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16392	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16393	AssertRCReturn(rc, rc);
16394
16395	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
16396	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
16397
16398	RTGCUINT uErrorCode = 0;
16399	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16400	uErrorCode \|= X86_TRAP_PF_ID;
16401	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16402	uErrorCode \|= X86_TRAP_PF_RW;
16403	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16404	uErrorCode \|= X86_TRAP_PF_P;
16405
16406	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16407
16408
16409	/* Handle the pagefault trap for the nested shadow table. */
16410	PVM pVM = pVCpu->CTX_SUFF(pVM);
16411	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16412
16413	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
16414	pCtx->cs.Sel, pCtx->rip));
16415
16416	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16417	TRPMResetTrap(pVCpu);
16418
16419	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16420	if ( rcStrict == VINF_SUCCESS
16421	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16422	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16423	{
16424	/* Successfully synced our nested page tables. */
16425	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16426	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16427	return VINF_SUCCESS;
16428	}
16429
16430	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16431	return rcStrict;
16432	}
16433
16434
16435	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16436	/**
16437	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16438	*/
16439	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16440	{
16441	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16442
16443	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16444	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16445	\| CPUMCTX_EXTRN_HWVIRT
16446	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16447	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16448	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16449	AssertRCReturn(rc, rc);
16450
16451	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16452
16453	VMXVEXITINFO ExitInfo;
16454	RT_ZERO(ExitInfo);
16455	ExitInfo.uReason = pVmxTransient->uExitReason;
16456	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16457	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16458	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16459	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16460
16461	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16462	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16463	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16464	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16465	{
16466	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16467	rcStrict = VINF_SUCCESS;
16468	}
16469	return rcStrict;
16470	}
16471
16472
16473	/**
16474	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16475	*/
16476	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16477	{
16478	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16479
16480	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16481	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16482	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16483	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16484	AssertRCReturn(rc, rc);
16485
16486	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16487
16488	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16489	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16490	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16491	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16492	{
16493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16494	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16495	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16496	}
16497	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16498	return rcStrict;
16499	}
16500
16501
16502	/**
16503	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16504	*/
16505	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16506	{
16507	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16508
16509	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16510	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16511	\| CPUMCTX_EXTRN_HWVIRT
16512	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16513	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16514	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16515	AssertRCReturn(rc, rc);
16516
16517	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16518
16519	VMXVEXITINFO ExitInfo;
16520	RT_ZERO(ExitInfo);
16521	ExitInfo.uReason = pVmxTransient->uExitReason;
16522	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16523	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16524	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16525	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16526
16527	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16528	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16529	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16530	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16531	{
16532	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16533	rcStrict = VINF_SUCCESS;
16534	}
16535	return rcStrict;
16536	}
16537
16538
16539	/**
16540	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16541	*/
16542	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16543	{
16544	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16545
16546	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16547	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16548	\| CPUMCTX_EXTRN_HWVIRT
16549	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16550	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16551	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16552	AssertRCReturn(rc, rc);
16553
16554	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16555
16556	VMXVEXITINFO ExitInfo;
16557	RT_ZERO(ExitInfo);
16558	ExitInfo.uReason = pVmxTransient->uExitReason;
16559	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16560	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16561	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16562	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16563
16564	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16565	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16566	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16567	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16568	{
16569	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16570	rcStrict = VINF_SUCCESS;
16571	}
16572	return rcStrict;
16573	}
16574
16575
16576	/**
16577	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16578	*/
16579	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16580	{
16581	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16582
16583	/*
16584	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16585	* thus might not need to import the shadow VMCS state, it's safer just in case
16586	* code elsewhere dares look at unsynced VMCS fields.
16587	*/
16588	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16589	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16590	\| CPUMCTX_EXTRN_HWVIRT
16591	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16592	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16593	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16594	AssertRCReturn(rc, rc);
16595
16596	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16597
16598	VMXVEXITINFO ExitInfo;
16599	RT_ZERO(ExitInfo);
16600	ExitInfo.uReason = pVmxTransient->uExitReason;
16601	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16602	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16603	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16604	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16605	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16606
16607	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16608	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16609	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16610	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16611	{
16612	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16613	rcStrict = VINF_SUCCESS;
16614	}
16615	return rcStrict;
16616	}
16617
16618
16619	/**
16620	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16621	*/
16622	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16623	{
16624	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16625
16626	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16627	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16628	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16629	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16630	AssertRCReturn(rc, rc);
16631
16632	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16633
16634	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16635	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16636	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16637	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16638	{
16639	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16640	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16641	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16642	}
16643	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16644	return rcStrict;
16645	}
16646
16647
16648	/**
16649	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16650	*/
16651	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16652	{
16653	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16654
16655	/*
16656	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16657	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16658	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16659	*/
16660	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16661	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16662	\| CPUMCTX_EXTRN_HWVIRT
16663	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16664	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16665	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16666	AssertRCReturn(rc, rc);
16667
16668	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16669
16670	VMXVEXITINFO ExitInfo;
16671	RT_ZERO(ExitInfo);
16672	ExitInfo.uReason = pVmxTransient->uExitReason;
16673	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16674	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16675	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16676	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16677	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16678
16679	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16680	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16681	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16682	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16683	{
16684	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16685	rcStrict = VINF_SUCCESS;
16686	}
16687	return rcStrict;
16688	}
16689
16690
16691	/**
16692	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16693	*/
16694	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16695	{
16696	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16697
16698	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16699	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16700	\| CPUMCTX_EXTRN_HWVIRT
16701	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16702	AssertRCReturn(rc, rc);
16703
16704	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16705
16706	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16707	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16708	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16709	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16710	{
16711	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16712	rcStrict = VINF_SUCCESS;
16713	}
16714	return rcStrict;
16715	}
16716
16717
16718	/**
16719	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16720	*/
16721	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16722	{
16723	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16724
16725	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16726	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16727	\| CPUMCTX_EXTRN_HWVIRT
16728	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16729	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16730	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16731	AssertRCReturn(rc, rc);
16732
16733	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16734
16735	VMXVEXITINFO ExitInfo;
16736	RT_ZERO(ExitInfo);
16737	ExitInfo.uReason = pVmxTransient->uExitReason;
16738	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16739	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16740	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16741	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16742
16743	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16744	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16745	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16746	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16747	{
16748	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16749	rcStrict = VINF_SUCCESS;
16750	}
16751	return rcStrict;
16752	}
16753
16754
16755	/**
16756	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16757	*/
16758	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16759	{
16760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16761
16762	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16763	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16764	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16765	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16766	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16767	AssertRCReturn(rc, rc);
16768
16769	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16770
16771	VMXVEXITINFO ExitInfo;
16772	RT_ZERO(ExitInfo);
16773	ExitInfo.uReason = pVmxTransient->uExitReason;
16774	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16775	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16776	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16777	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16778
16779	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16780	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16781	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16782	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16783	{
16784	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16785	rcStrict = VINF_SUCCESS;
16786	}
16787	return rcStrict;
16788	}
16789	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16790	/** @} */
16791
16792
16793	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16794	/** @name Nested-guest VM-exit handlers.
16795	* @{
16796	*/
16797	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16798	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16799	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16800
16801	/**
16802	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16803	* Conditional VM-exit.
16804	*/
16805	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16806	{
16807	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16808
16809	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16810	AssertRCReturn(rc, rc);
16811
16812	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16813	uint32_t const uExtIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16814	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16815
16816	/*
16817	* Make sure not to use stale/previous VM-exit instruction length since we read the
16818	* instruction length from the VMCS below only for software exceptions and privileged
16819	* software exceptions but we pass it for all exception VM-exits below.
16820	*/
16821	pVmxTransient->cbInstr = 0;
16822
16823	switch (uExtIntType)
16824	{
16825	/*
16826	* Physical NMIs:
16827	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16828	*/
16829	case VMX_EXIT_INT_INFO_TYPE_NMI:
16830	return hmR0VmxExitHostNmi(pVCpu);
16831
16832	/*
16833	* Hardware exceptions,
16834	* Software exceptions,
16835	* Privileged software exceptions:
16836	* Figure out if the exception must be delivered to the guest or the nested-guest.
16837	*
16838	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16839	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16840	* length.
16841	*/
16842	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16843	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16844	{
16845	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16846	AssertRCReturn(rc, rc);
16847	RT_FALL_THRU();
16848	}
16849	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16850	{
16851	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16852	AssertRCReturn(rc, rc);
16853
16854	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
16855	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uVector,
16856	pVmxTransient->uExitIntErrorCode);
16857	if (fIntercept)
16858	{
16859	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16860	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16861	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16862	AssertRCReturn(rc, rc);
16863
16864	VMXVEXITINFO ExitInfo;
16865	RT_ZERO(ExitInfo);
16866	ExitInfo.uReason = pVmxTransient->uExitReason;
16867	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16868	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16869
16870	VMXVEXITEVENTINFO ExitEventInfo;
16871	RT_ZERO(ExitEventInfo);
16872	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16873	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16874	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16875	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16876
16877	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16878	}
16879
16880	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs. */
16881	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16882
16883	/* If the guest hypervisor is not intercepting the exception, forward it to the guest. */
16884	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo), pVmxTransient->cbInstr,
16885	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
16886	return VINF_SUCCESS;
16887	}
16888
16889	/*
16890	* Software interrupts:
16891	* VM-exits cannot be caused by software interrupts.
16892	*
16893	* External interrupts:
16894	* This should only happen when "acknowledge external interrupts on VM-exit"
16895	* control is set. However, we never set this when executing a guest or
16896	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16897	* the guest.
16898	*/
16899	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16900	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16901	default:
16902	{
16903	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16904	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16905	}
16906	}
16907	}
16908
16909
16910	/**
16911	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16912	* Unconditional VM-exit.
16913	*/
16914	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16915	{
16916	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16917	return IEMExecVmxVmexitTripleFault(pVCpu);
16918	}
16919
16920
16921	/**
16922	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16923	*/
16924	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16925	{
16926	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16927
16928	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16929	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16930	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16931	}
16932
16933
16934	/**
16935	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16936	*/
16937	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16938	{
16939	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16940
16941	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16942	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16943	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16944	}
16945
16946
16947	/**
16948	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16949	* Unconditional VM-exit.
16950	*/
16951	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16952	{
16953	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16954
16955	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16956	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16957	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16958	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16959	AssertRCReturn(rc, rc);
16960
16961	VMXVEXITINFO ExitInfo;
16962	RT_ZERO(ExitInfo);
16963	ExitInfo.uReason = pVmxTransient->uExitReason;
16964	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16965	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16966
16967	VMXVEXITEVENTINFO ExitEventInfo;
16968	RT_ZERO(ExitEventInfo);
16969	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16970	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16971	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16972	}
16973
16974
16975	/**
16976	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16977	*/
16978	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16979	{
16980	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16981
16982	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16983	{
16984	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16985	AssertRCReturn(rc, rc);
16986	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16987	}
16988	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16989	}
16990
16991
16992	/**
16993	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16994	*/
16995	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16996	{
16997	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16998
16999	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17000	{
17001	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17002	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17003	AssertRCReturn(rc, rc);
17004
17005	VMXVEXITINFO ExitInfo;
17006	RT_ZERO(ExitInfo);
17007	ExitInfo.uReason = pVmxTransient->uExitReason;
17008	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17009	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17010	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17011	}
17012	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
17013	}
17014
17015
17016	/**
17017	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
17018	*/
17019	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17020	{
17021	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17022
17023	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
17024	{
17025	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17026	AssertRCReturn(rc, rc);
17027	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17028	}
17029	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
17030	}
17031
17032
17033	/**
17034	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
17035	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
17036	*/
17037	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17038	{
17039	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17040
17041	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
17042	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
17043
17044	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17045	AssertRCReturn(rc, rc);
17046
17047	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
17048	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17049	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17050
17051	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
17052	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
17053	u64VmcsField &= UINT64_C(0xffffffff);
17054
17055	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
17056	{
17057	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17058	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17059	AssertRCReturn(rc, rc);
17060
17061	VMXVEXITINFO ExitInfo;
17062	RT_ZERO(ExitInfo);
17063	ExitInfo.uReason = pVmxTransient->uExitReason;
17064	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17065	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17066	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17067	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17068	}
17069
17070	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
17071	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
17072	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
17073	}
17074
17075
17076	/**
17077	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
17078	*/
17079	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17080	{
17081	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17082
17083	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17084	{
17085	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17086	AssertRCReturn(rc, rc);
17087	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17088	}
17089
17090	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
17091	}
17092
17093
17094	/**
17095	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
17096	* Conditional VM-exit.
17097	*/
17098	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17099	{
17100	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17101
17102	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17103	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17104	AssertRCReturn(rc, rc);
17105
17106	VBOXSTRICTRC rcStrict;
17107	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
17108	switch (uAccessType)
17109	{
17110	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
17111	{
17112	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17113	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17114	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
17115	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
17116
17117	bool fIntercept;
17118	switch (iCrReg)
17119	{
17120	case 0:
17121	case 4:
17122	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
17123	break;
17124
17125	case 3:
17126	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
17127	break;
17128
17129	case 8:
17130	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
17131	break;
17132
17133	default:
17134	fIntercept = false;
17135	break;
17136	}
17137	if (fIntercept)
17138	{
17139	VMXVEXITINFO ExitInfo;
17140	RT_ZERO(ExitInfo);
17141	ExitInfo.uReason = pVmxTransient->uExitReason;
17142	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17143	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17144	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17145	}
17146	else
17147	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17148	break;
17149	}
17150
17151	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
17152	{
17153	/*
17154	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
17155	* CR2 reads do not cause a VM-exit.
17156	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
17157	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
17158	*/
17159	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
17160	if ( iCrReg == 3
17161	\|\| iCrReg == 8)
17162	{
17163	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
17164	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
17165	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
17166	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
17167	{
17168	VMXVEXITINFO ExitInfo;
17169	RT_ZERO(ExitInfo);
17170	ExitInfo.uReason = pVmxTransient->uExitReason;
17171	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17172	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17173	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17174	}
17175	else
17176	{
17177	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
17178	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
17179	}
17180	}
17181	else
17182	{
17183	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
17184	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
17185	}
17186	break;
17187	}
17188
17189	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
17190	{
17191	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
17192	Assert(pVmcsNstGst);
17193	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
17194	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
17195	if ( (uGstHostMask & X86_CR0_TS)
17196	&& (uReadShadow & X86_CR0_TS))
17197	{
17198	VMXVEXITINFO ExitInfo;
17199	RT_ZERO(ExitInfo);
17200	ExitInfo.uReason = pVmxTransient->uExitReason;
17201	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17202	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17203	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17204	}
17205	else
17206	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
17207	break;
17208	}
17209
17210	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
17211	{
17212	RTGCPTR GCPtrEffDst;
17213	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
17214	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
17215	if (fMemOperand)
17216	{
17217	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17218	AssertRCReturn(rc, rc);
17219	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
17220	}
17221	else
17222	GCPtrEffDst = NIL_RTGCPTR;
17223
17224	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
17225	{
17226	VMXVEXITINFO ExitInfo;
17227	RT_ZERO(ExitInfo);
17228	ExitInfo.uReason = pVmxTransient->uExitReason;
17229	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17230	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
17231	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17232	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17233	}
17234	else
17235	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
17236	break;
17237	}
17238
17239	default:
17240	{
17241	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
17242	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
17243	}
17244	}
17245
17246	if (rcStrict == VINF_IEM_RAISED_XCPT)
17247	{
17248	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
17249	rcStrict = VINF_SUCCESS;
17250	}
17251	return rcStrict;
17252	}
17253
17254
17255	/**
17256	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
17257	* Conditional VM-exit.
17258	*/
17259	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17260	{
17261	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17262
17263	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
17264	{
17265	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17266	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17267	AssertRCReturn(rc, rc);
17268
17269	VMXVEXITINFO ExitInfo;
17270	RT_ZERO(ExitInfo);
17271	ExitInfo.uReason = pVmxTransient->uExitReason;
17272	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17273	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17274	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17275	}
17276	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
17277	}
17278
17279
17280	/**
17281	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
17282	* Conditional VM-exit.
17283	*/
17284	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17285	{
17286	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17287
17288	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17289	AssertRCReturn(rc, rc);
17290
17291	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
17292	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
17293	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
17294
17295	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
17296	uint8_t const cbAccess = s_aIOSizes[uIOSize];
17297	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
17298	{
17299	/*
17300	* IN/OUT instruction:
17301	* - Provides VM-exit instruction length.
17302	*
17303	* INS/OUTS instruction:
17304	* - Provides VM-exit instruction length.
17305	* - Provides Guest-linear address.
17306	* - Optionally provides VM-exit instruction info (depends on CPU feature).
17307	*/
17308	PVM pVM = pVCpu->CTX_SUFF(pVM);
17309	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17310	AssertRCReturn(rc, rc);
17311
17312	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
17313	pVmxTransient->ExitInstrInfo.u = 0;
17314	pVmxTransient->uGuestLinearAddr = 0;
17315
17316	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
17317	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
17318	if (fIOString)
17319	{
17320	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
17321	if (fVmxInsOutsInfo)
17322	{
17323	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
17324	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17325	}
17326	}
17327	AssertRCReturn(rc, rc);
17328
17329	VMXVEXITINFO ExitInfo;
17330	RT_ZERO(ExitInfo);
17331	ExitInfo.uReason = pVmxTransient->uExitReason;
17332	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17333	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17334	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17335	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
17336	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17337	}
17338	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
17339	}
17340
17341
17342	/**
17343	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
17344	*/
17345	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17346	{
17347	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17348
17349	uint32_t fMsrpm;
17350	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17351	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17352	else
17353	fMsrpm = VMXMSRPM_EXIT_RD;
17354
17355	if (fMsrpm & VMXMSRPM_EXIT_RD)
17356	{
17357	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17358	AssertRCReturn(rc, rc);
17359	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17360	}
17361	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
17362	}
17363
17364
17365	/**
17366	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
17367	*/
17368	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17369	{
17370	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17371
17372	uint32_t fMsrpm;
17373	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17374	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17375	else
17376	fMsrpm = VMXMSRPM_EXIT_WR;
17377
17378	if (fMsrpm & VMXMSRPM_EXIT_WR)
17379	{
17380	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17381	AssertRCReturn(rc, rc);
17382	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17383	}
17384	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17385	}
17386
17387
17388	/**
17389	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17390	*/
17391	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17392	{
17393	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17394
17395	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17396	{
17397	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17398	AssertRCReturn(rc, rc);
17399	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17400	}
17401	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17402	}
17403
17404
17405	/**
17406	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17407	* VM-exit.
17408	*/
17409	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17410	{
17411	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17412
17413	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17414	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17415	}
17416
17417
17418	/**
17419	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17420	*/
17421	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17422	{
17423	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17424
17425	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17426	{
17427	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17428	AssertRCReturn(rc, rc);
17429	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17430	}
17431	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17432	}
17433
17434
17435	/**
17436	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17437	*/
17438	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17439	{
17440	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17441
17442	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17443	* PAUSE when executing a nested-guest? If it does not, we would not need
17444	* to check for the intercepts here. Just call VM-exit... */
17445
17446	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17447	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17448	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17449	{
17450	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17451	AssertRCReturn(rc, rc);
17452	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17453	}
17454	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17455	}
17456
17457
17458	/**
17459	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17460	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17461	*/
17462	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17463	{
17464	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17465
17466	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17467	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17468	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17469	}
17470
17471
17472	/**
17473	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17474	* VM-exit.
17475	*/
17476	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17477	{
17478	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17479
17480	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17481	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17482	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17483	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17484	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17485	AssertRCReturn(rc, rc);
17486
17487	VMXVEXITINFO ExitInfo;
17488	RT_ZERO(ExitInfo);
17489	ExitInfo.uReason = pVmxTransient->uExitReason;
17490	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17491	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17492
17493	VMXVEXITEVENTINFO ExitEventInfo;
17494	RT_ZERO(ExitEventInfo);
17495	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17496	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17497	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17498	}
17499
17500
17501	/**
17502	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17503	* Conditional VM-exit.
17504	*/
17505	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17506	{
17507	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17508
17509	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17510	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17511	AssertRCReturn(rc, rc);
17512
17513	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17514	}
17515
17516
17517	/**
17518	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17519	* Conditional VM-exit.
17520	*/
17521	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17522	{
17523	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17524
17525	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17526	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17527	AssertRCReturn(rc, rc);
17528
17529	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17530	}
17531
17532
17533	/**
17534	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17535	*/
17536	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17537	{
17538	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17539
17540	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17541	{
17542	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17543	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17544	AssertRCReturn(rc, rc);
17545	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17546	}
17547	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17548	}
17549
17550
17551	/**
17552	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17553	*/
17554	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17555	{
17556	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17557
17558	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17559	{
17560	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17561	AssertRCReturn(rc, rc);
17562	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17563	}
17564	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17565	}
17566
17567
17568	/**
17569	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17570	*/
17571	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17572	{
17573	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17574
17575	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17576	{
17577	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17578	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17579	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17580	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17581	AssertRCReturn(rc, rc);
17582
17583	VMXVEXITINFO ExitInfo;
17584	RT_ZERO(ExitInfo);
17585	ExitInfo.uReason = pVmxTransient->uExitReason;
17586	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17587	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17588	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17589	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17590	}
17591	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17592	}
17593
17594
17595	/**
17596	* Nested-guest VM-exit handler for invalid-guest state
17597	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17598	*/
17599	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17600	{
17601	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17602
17603	/*
17604	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17605	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17606	* Handle it like it's in an invalid guest state of the outer guest.
17607	*
17608	* When the fast path is implemented, this should be changed to cause the corresponding
17609	* nested-guest VM-exit.
17610	*/
17611	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17612	}
17613
17614
17615	/**
17616	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17617	* and only provide the instruction length.
17618	*
17619	* Unconditional VM-exit.
17620	*/
17621	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17622	{
17623	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17624
17625	#ifdef VBOX_STRICT
17626	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17627	switch (pVmxTransient->uExitReason)
17628	{
17629	case VMX_EXIT_ENCLS:
17630	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17631	break;
17632
17633	case VMX_EXIT_VMFUNC:
17634	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17635	break;
17636	}
17637	#endif
17638
17639	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17640	AssertRCReturn(rc, rc);
17641	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17642	}
17643
17644
17645	/**
17646	* Nested-guest VM-exit handler for instructions that provide instruction length as
17647	* well as more information.
17648	*
17649	* Unconditional VM-exit.
17650	*/
17651	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17652	{
17653	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17654
17655	#ifdef VBOX_STRICT
17656	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17657	switch (pVmxTransient->uExitReason)
17658	{
17659	case VMX_EXIT_GDTR_IDTR_ACCESS:
17660	case VMX_EXIT_LDTR_TR_ACCESS:
17661	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17662	break;
17663
17664	case VMX_EXIT_RDRAND:
17665	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17666	break;
17667
17668	case VMX_EXIT_RDSEED:
17669	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17670	break;
17671
17672	case VMX_EXIT_XSAVES:
17673	case VMX_EXIT_XRSTORS:
17674	/** @todo NSTVMX: Verify XSS-bitmap. */
17675	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17676	break;
17677
17678	case VMX_EXIT_UMWAIT:
17679	case VMX_EXIT_TPAUSE:
17680	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17681	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17682	break;
17683	}
17684	#endif
17685
17686	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17687	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17688	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17689	AssertRCReturn(rc, rc);
17690
17691	VMXVEXITINFO ExitInfo;
17692	RT_ZERO(ExitInfo);
17693	ExitInfo.uReason = pVmxTransient->uExitReason;
17694	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17695	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17696	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17697	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17698	}
17699
17700	/** @} */
17701	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17702

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 79637

Download in other formats: